public StringDimensionMergerV9( String dimensionName, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, ProgressIndicator progress, Closer closer ) { this.dimensionName = dimensionName; this.indexSpec = indexSpec; this.capabilities = capabilities; this.segmentWriteOutMedium = segmentWriteOutMedium; nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); this.progress = progress; this.closer = closer; }
static GenericColumnSerializer createFloatColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec ) { // If using default values for null use FloatColumnSerializer to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { return FloatColumnSerializer.create( segmentWriteOutMedium, columnName, indexSpec.getMetricCompression() ); } else { return FloatColumnSerializerV2.create( segmentWriteOutMedium, columnName, indexSpec.getMetricCompression(), indexSpec.getBitmapSerdeFactory() ); } }
static GenericColumnSerializer createDoubleColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec ) { // If using default values for null use DoubleColumnSerializer to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { return DoubleColumnSerializer.create( segmentWriteOutMedium, columnName, indexSpec.getMetricCompression() ); } else { return DoubleColumnSerializerV2.create( segmentWriteOutMedium, columnName, indexSpec.getMetricCompression(), indexSpec.getBitmapSerdeFactory() ); } }
GenericIndexed<String> dims = GenericIndexed.fromIterable(finalDimensions, GenericIndexed.STRING_STRATEGY); final String bitmapSerdeFactoryType = mapper.writeValueAsString(indexSpec.getBitmapSerdeFactory()); final long numBytes = cols.getSerializedSize() + dims.getSerializedSize()
@Before public void setUp() throws IndexSizeExceededException { long timestamp = 0L; for (Map<String, Object> event : events1) { incrementalIndex1.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event)); } timestamp = 0L; for (Map<String, Object> event : events2) { incrementalIndex2.add(new MapBasedInputRow(timestamp++, Lists.newArrayList(event.keySet()), event)); } adapter2 = new IncrementalIndexAdapter( DEFAULT_INTERVAL, incrementalIndex2, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory() ); adapter1 = new IncrementalIndexAdapter( DEFAULT_INTERVAL, incrementalIndex1, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory() ); }
static GenericColumnSerializer createLongColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec ) { // If using default values for null use LongColumnSerializer to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { return LongColumnSerializer.create( segmentWriteOutMedium, columnName, indexSpec.getMetricCompression(), indexSpec.getLongEncoding() ); } else { return LongColumnSerializerV2.create( segmentWriteOutMedium, columnName, indexSpec.getMetricCompression(), indexSpec.getLongEncoding(), indexSpec.getBitmapSerdeFactory() ); } }
dataInterval, index, indexSpec.getBitmapSerdeFactory().getBitmapFactory()
static ColumnPartSerde createDoubleColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) { // If using default values for null use DoubleNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { return DoubleNumericColumnPartSerde.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate(serializer) .build(); } else { return DoubleNumericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .withDelegate(serializer) .build(); } }
@Test public void testSerde() throws Exception { final ObjectMapper objectMapper = new DefaultObjectMapper(); final String json = "{ \"bitmap\" : { \"type\" : \"roaring\" }, \"dimensionCompression\" : \"lz4\", \"metricCompression\" : \"lzf\"" + ", \"longEncoding\" : \"auto\" }"; final IndexSpec spec = objectMapper.readValue(json, IndexSpec.class); Assert.assertEquals(new RoaringBitmapSerdeFactory(null), spec.getBitmapSerdeFactory()); Assert.assertEquals(CompressionStrategy.LZ4, spec.getDimensionCompression()); Assert.assertEquals(CompressionStrategy.LZF, spec.getMetricCompression()); Assert.assertEquals(CompressionFactory.LongEncodingStrategy.AUTO, spec.getLongEncoding()); Assert.assertEquals(spec, objectMapper.readValue(objectMapper.writeValueAsBytes(spec), IndexSpec.class)); }
@Test public void testGetRowsIterable() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex toPersist1 = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, toPersist1); final IndexableAdapter incrementalAdapter = new IncrementalIndexAdapter( toPersist1.getInterval(), toPersist1, INDEX_SPEC.getBitmapSerdeFactory() .getBitmapFactory() ); RowIterator rows = incrementalAdapter.getRows(); List<Integer> rowNums = new ArrayList<>(); while (rows.moveToNext()) { rowNums.add(rows.getPointer().getRowNum()); } Assert.assertEquals(2, rowNums.size()); Assert.assertEquals(0, (long) rowNums.get(0)); Assert.assertEquals(1, (long) rowNums.get(1)); }
static ColumnPartSerde createLongColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) { // If using default values for null use LongNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { return LongNumericColumnPartSerde.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate(serializer) .build(); } else { return LongNumericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .withDelegate(serializer) .build(); } }
static ColumnPartSerde createFloatColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) { // If using default values for null use FloatNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { return FloatNumericColumnPartSerde.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate(serializer) .build(); } else { return FloatNumericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .withDelegate(serializer) .build(); } }
final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); segmentWriteOutMedium, bmpFilename, indexSpec.getBitmapSerdeFactory().getObjectStrategy() ); bitmapWriter.open();
@Test public void testGetBitmapIndex() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex incrementalIndex = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, incrementalIndex); IndexableAdapter adapter = new IncrementalIndexAdapter( incrementalIndex.getInterval(), incrementalIndex, INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory() ); String dimension = "dim1"; try (CloseableIndexed<String> dimValueLookup = adapter.getDimValueLookup(dimension)) { for (int i = 0; i < dimValueLookup.size(); i++) { BitmapValues bitmapValues = adapter.getBitmapValues(dimension, i); Assert.assertEquals(1, bitmapValues.size()); } } }
@Override public ColumnDescriptor makeColumnDescriptor() { // Now write everything boolean hasMultiValue = capabilities.hasMultipleValues(); final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.STRING); builder.setHasMultipleValues(hasMultiValue); final DictionaryEncodedColumnPartSerde.SerializerBuilder partBuilder = DictionaryEncodedColumnPartSerde .serializerBuilder() .withDictionary(dictionaryWriter) .withValue( encodedValueSerializer, hasMultiValue, compressionStrategy != CompressionStrategy.UNCOMPRESSED ) .withBitmapSerdeFactory(bitmapSerdeFactory) .withBitmapIndex(bitmapWriter) .withSpatialIndex(spatialWriter) .withByteOrder(IndexIO.BYTE_ORDER); final ColumnDescriptor serdeficator = builder .addSerde(partBuilder.build()) .build(); //log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime); return serdeficator; }
toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory() .getBitmapFactory() );
toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory() .getBitmapFactory() ); indexSpec.getBitmapSerdeFactory(), CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF :
toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory() .getBitmapFactory() );
toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory() .getBitmapFactory() );
toPersist1.getInterval(), toPersist1, indexSpec.getBitmapSerdeFactory() .getBitmapFactory() ); indexSpec.getBitmapSerdeFactory(), CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF :