@Override public boolean hasMultipleValues(final String dimension) { if (isVirtualColumn(dimension)) { return virtualColumns.getVirtualColumn(dimension).capabilities(dimension).hasMultipleValues(); } final ColumnHolder columnHolder = index.getColumnHolder(dimension); return columnHolder != null && columnHolder.getCapabilities().hasMultipleValues(); }
static boolean supportsSelectivityEstimation( Filter filter, String dimension, ColumnSelector columnSelector, BitmapIndexSelector indexSelector ) { if (filter.supportsBitmapIndex(indexSelector)) { final ColumnHolder columnHolder = columnSelector.getColumnHolder(dimension); if (columnHolder != null) { return !columnHolder.getCapabilities().hasMultipleValues(); } } return false; }
private ColumnAnalysis analyzeNumericColumn( final ColumnCapabilities capabilities, final int length, final int sizePerRow ) { long size = 0; if (analyzingSize()) { if (capabilities.hasMultipleValues()) { return ColumnAnalysis.error("multi_value"); } size = ((long) length) * sizePerRow; } return new ColumnAnalysis( capabilities.getType().name(), capabilities.hasMultipleValues(), size, null, null, null, null ); }
final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues(); long size = 0;
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
public void merge(ColumnCapabilities other) { if (other == null) { return; } if (type == null) { type = other.getType(); } if (!type.equals(other.getType())) { throw new ISE("Cannot merge columns of type[%s] and [%s]", type, other.getType()); } this.dictionaryEncoded |= other.isDictionaryEncoded(); this.runLengthEncoded |= other.isRunLengthEncoded(); this.hasInvertedIndexes |= other.hasBitmapIndexes(); this.hasSpatialIndexes |= other.hasSpatialIndexes(); this.hasMultipleValues |= other.hasMultipleValues(); this.filterable &= other.isFilterable(); } }
.allMatch(dimension -> { final ColumnCapabilities columnCapabilities = storageAdapter.getColumnCapabilities(dimension.getDimension()); return columnCapabilities != null && !columnCapabilities.hasMultipleValues(); });
@Override public ColumnDescriptor makeColumnDescriptor() { // Now write everything boolean hasMultiValue = capabilities.hasMultipleValues(); final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.STRING); builder.setHasMultipleValues(hasMultiValue); final DictionaryEncodedColumnPartSerde.SerializerBuilder partBuilder = DictionaryEncodedColumnPartSerde .serializerBuilder() .withDictionary(dictionaryWriter) .withValue( encodedValueSerializer, hasMultiValue, compressionStrategy != CompressionStrategy.UNCOMPRESSED ) .withBitmapSerdeFactory(bitmapSerdeFactory) .withBitmapIndex(bitmapWriter) .withSpatialIndex(spatialWriter) .withByteOrder(IndexIO.BYTE_ORDER); final ColumnDescriptor serdeficator = builder .addSerde(partBuilder.build()) .build(); //log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime); return serdeficator; }
@Test public void testSerde() throws Exception { String json = mapper.writeValueAsString(new ColumnCapabilitiesImpl() .setDictionaryEncoded(true) .setHasBitmapIndexes(true) .setHasMultipleValues(true) .setHasSpatialIndexes(true) .setType(ValueType.COMPLEX) .setFilterable(true)); Assert.assertFalse(json.contains("filterable")); ColumnCapabilities cc = mapper.readValue(json, ColumnCapabilitiesImpl.class); Assert.assertEquals(ValueType.COMPLEX, cc.getType()); Assert.assertTrue(cc.isDictionaryEncoded()); Assert.assertFalse(cc.isRunLengthEncoded()); Assert.assertTrue(cc.hasSpatialIndexes()); Assert.assertTrue(cc.hasMultipleValues()); Assert.assertTrue(cc.hasBitmapIndexes()); Assert.assertFalse(cc.isFilterable()); }
capabilities.hasMultipleValues(), size, cardinality,
@Test public void testDeserialization() throws Exception { String json = "{\n" + " \"type\":\"COMPLEX\",\n" + " \"dictionaryEncoded\":true,\n" + " \"runLengthEncoded\":true,\n" + " \"hasSpatialIndexes\":true,\n" + " \"hasMultipleValues\":true,\n" + " \"hasBitmapIndexes\":true,\n" + " \"filterable\":true\n" + "}"; ColumnCapabilities cc = mapper.readValue(json, ColumnCapabilitiesImpl.class); Assert.assertEquals(ValueType.COMPLEX, cc.getType()); Assert.assertTrue(cc.isDictionaryEncoded()); Assert.assertTrue(cc.isRunLengthEncoded()); Assert.assertTrue(cc.hasSpatialIndexes()); Assert.assertTrue(cc.hasMultipleValues()); Assert.assertTrue(cc.hasBitmapIndexes()); Assert.assertFalse(cc.isFilterable()); } }
capabilities.hasMultipleValues(), size, analyzingCardinality() ? cardinality : 0,
@Override public boolean hasMultipleValues(final String dimension) { if (isVirtualColumn(dimension)) { return virtualColumns.getVirtualColumn(dimension).capabilities(dimension).hasMultipleValues(); } final ColumnHolder columnHolder = index.getColumnHolder(dimension); return columnHolder != null && columnHolder.getCapabilities().hasMultipleValues(); }
private ColumnAnalysis analyzeNumericColumn( final ColumnCapabilities capabilities, final int length, final int sizePerRow ) { long size = 0; if (analyzingSize()) { if (capabilities.hasMultipleValues()) { return ColumnAnalysis.error("multi_value"); } size = ((long) length) * sizePerRow; } return new ColumnAnalysis( capabilities.getType().name(), capabilities.hasMultipleValues(), size, null, null, null, null ); }
static boolean supportsSelectivityEstimation( Filter filter, String dimension, ColumnSelector columnSelector, BitmapIndexSelector indexSelector ) { if (filter.supportsBitmapIndex(indexSelector)) { final ColumnHolder columnHolder = columnSelector.getColumnHolder(dimension); if (columnHolder != null) { return !columnHolder.getCapabilities().hasMultipleValues(); } } return false; }
final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues(); long size = 0;
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
public void merge(ColumnCapabilities other) { if (other == null) { return; } if (type == null) { type = other.getType(); } if (!type.equals(other.getType())) { throw new ISE("Cannot merge columns of type[%s] and [%s]", type, other.getType()); } this.dictionaryEncoded |= other.isDictionaryEncoded(); this.runLengthEncoded |= other.isRunLengthEncoded(); this.hasInvertedIndexes |= other.hasBitmapIndexes(); this.hasSpatialIndexes |= other.hasSpatialIndexes(); this.hasMultipleValues |= other.hasMultipleValues(); this.filterable &= other.isFilterable(); } }
@Override public ColumnDescriptor makeColumnDescriptor() { // Now write everything boolean hasMultiValue = capabilities.hasMultipleValues(); final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.STRING); builder.setHasMultipleValues(hasMultiValue); final DictionaryEncodedColumnPartSerde.SerializerBuilder partBuilder = DictionaryEncodedColumnPartSerde .serializerBuilder() .withDictionary(dictionaryWriter) .withValue( encodedValueSerializer, hasMultiValue, compressionStrategy != CompressionStrategy.UNCOMPRESSED ) .withBitmapSerdeFactory(bitmapSerdeFactory) .withBitmapIndex(bitmapWriter) .withSpatialIndex(spatialWriter) .withByteOrder(IndexIO.BYTE_ORDER); final ColumnDescriptor serdeficator = builder .addSerde(partBuilder.build()) .build(); //log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime); return serdeficator; }
capabilities.hasMultipleValues(), size, analyzingCardinality() ? cardinality : 0,