@Test public void testSerdeUncompressed() throws Exception { final ObjectMapper objectMapper = new DefaultObjectMapper(); final String json = "{ \"dimensionCompression\" : \"uncompressed\" }"; final IndexSpec spec = objectMapper.readValue(json, IndexSpec.class); Assert.assertEquals(CompressionStrategy.UNCOMPRESSED, spec.getDimensionCompression()); Assert.assertEquals(spec, objectMapper.readValue(objectMapper.writeValueAsBytes(spec), IndexSpec.class)); }
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
@Test public void testSerde() throws Exception { final ObjectMapper objectMapper = new DefaultObjectMapper(); final String json = "{ \"bitmap\" : { \"type\" : \"roaring\" }, \"dimensionCompression\" : \"lz4\", \"metricCompression\" : \"lzf\"" + ", \"longEncoding\" : \"auto\" }"; final IndexSpec spec = objectMapper.readValue(json, IndexSpec.class); Assert.assertEquals(new RoaringBitmapSerdeFactory(null), spec.getBitmapSerdeFactory()); Assert.assertEquals(CompressionStrategy.LZ4, spec.getDimensionCompression()); Assert.assertEquals(CompressionStrategy.LZF, spec.getMetricCompression()); Assert.assertEquals(CompressionFactory.LongEncodingStrategy.AUTO, spec.getLongEncoding()); Assert.assertEquals(spec, objectMapper.readValue(objectMapper.writeValueAsBytes(spec), IndexSpec.class)); }
@Test public void testDefaults() { final IndexSpec spec = new IndexSpec(); Assert.assertEquals(CompressionStrategy.LZ4, spec.getDimensionCompression()); Assert.assertEquals(CompressionStrategy.LZ4, spec.getMetricCompression()); Assert.assertEquals(CompressionFactory.LongEncodingStrategy.LONGS, spec.getLongEncoding()); } }
Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression());
@Test public void testPersistWithSegmentMetadata() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, toPersist); Map<String, Object> metadataElems = ImmutableMap.of("key", "value"); toPersist.getMetadata().putAll(metadataElems); final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression()); Assert.assertEquals( new Metadata( metadataElems, IncrementalIndexTest.getDefaultCombiningAggregatorFactories(), null, Granularities.NONE, Boolean.TRUE ), index.getMetadata() ); }
Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(merged.getAvailableDimensions())); assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(index2, indexSpec.getDimensionCompression()); assertDimCompression(merged, indexSpec.getDimensionCompression());
Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression());
@Override public ColumnDescriptor makeColumnDescriptor() { // Now write everything boolean hasMultiValue = capabilities.hasMultipleValues(); final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.STRING); builder.setHasMultipleValues(hasMultiValue); final DictionaryEncodedColumnPartSerde.SerializerBuilder partBuilder = DictionaryEncodedColumnPartSerde .serializerBuilder() .withDictionary(dictionaryWriter) .withValue( encodedValueSerializer, hasMultiValue, compressionStrategy != CompressionStrategy.UNCOMPRESSED ) .withBitmapSerdeFactory(bitmapSerdeFactory) .withBitmapIndex(bitmapWriter) .withSpatialIndex(spatialWriter) .withByteOrder(IndexIO.BYTE_ORDER); final ColumnDescriptor serdeficator = builder .addSerde(partBuilder.build()) .build(); //log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime); return serdeficator; }
CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(merged, newSpec.getDimensionCompression());
@Test public void testPersist() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, toPersist); final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression()); Assert.assertArrayEquals( IncrementalIndexTest.getDefaultCombiningAggregatorFactories(), index.getMetadata().getAggregators() ); Assert.assertEquals( Granularities.NONE, index.getMetadata().getQueryGranularity() ); }
assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(merged, indexSpec.getDimensionCompression());
Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); assertDimCompression(index2, indexSpec.getDimensionCompression()); assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(merged, indexSpec.getDimensionCompression());
CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? CompressionStrategy.LZF : CompressionStrategy.LZ4, assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(converted, newSpec.getDimensionCompression());
assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(merged, indexSpec.getDimensionCompression());
assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(converted, indexSpec.getDimensionCompression());
protected void setupEncodedValueWriter() throws IOException { final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = V3CompressedVSizeColumnarMultiIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarMultiIntsSerializer(segmentWriteOutMedium, cardinality); } } else { if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { encodedValueSerializer = CompressedVSizeColumnarIntsSerializer.create( segmentWriteOutMedium, filenameBase, cardinality, compressionStrategy ); } else { encodedValueSerializer = new VSizeColumnarIntsSerializer(segmentWriteOutMedium, cardinality); } } encodedValueSerializer.open(); }
@Override public ColumnDescriptor makeColumnDescriptor() { // Now write everything boolean hasMultiValue = capabilities.hasMultipleValues(); final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.STRING); builder.setHasMultipleValues(hasMultiValue); final DictionaryEncodedColumnPartSerde.SerializerBuilder partBuilder = DictionaryEncodedColumnPartSerde .serializerBuilder() .withDictionary(dictionaryWriter) .withValue( encodedValueSerializer, hasMultiValue, compressionStrategy != CompressionStrategy.UNCOMPRESSED ) .withBitmapSerdeFactory(bitmapSerdeFactory) .withBitmapIndex(bitmapWriter) .withSpatialIndex(spatialWriter) .withByteOrder(IndexIO.BYTE_ORDER); final ColumnDescriptor serdeficator = builder .addSerde(partBuilder.build()) .build(); //log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime); return serdeficator; }