@Override public ColumnCapabilities getCapabilities(String column) { return input.getColumnHolder(column).getCapabilities(); }
private List<String> getColumnsToInclude(final QueryableIndex index) { final Set<String> columnNames = Sets.newLinkedHashSet(columnNamesFromCli); // Empty columnNames => include all columns. if (columnNames.isEmpty()) { columnNames.add(ColumnHolder.TIME_COLUMN_NAME); Iterables.addAll(columnNames, index.getColumnNames()); } else { // Remove any provided columns that do not exist in this segment. for (String columnName : ImmutableList.copyOf(columnNames)) { if (index.getColumnHolder(columnName) == null) { columnNames.remove(columnName); } } } return ImmutableList.copyOf(columnNames); }
@Override public int getDimensionCardinality(String dimension) { ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null) { return 0; } try (BaseColumn col = columnHolder.getColumn()) { if (!(col instanceof DictionaryEncodedColumn)) { return Integer.MAX_VALUE; } return ((DictionaryEncodedColumn) col).getCardinality(); } catch (IOException e) { throw new UncheckedIOException(e); } }
@Override public DateTime getMinTime() { try (final NumericColumn column = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return DateTimes.utc(column.getLongSingleValueRow(0)); } }
@VisibleForTesting BitmapValues getBitmapIndex(String dimension, String value) { final ColumnHolder columnHolder = input.getColumnHolder(dimension); if (columnHolder == null) { return BitmapValues.EMPTY; } final BitmapIndex bitmaps = columnHolder.getBitmapIndex(); if (bitmaps == null) { return BitmapValues.EMPTY; } return new ImmutableBitmapValues(bitmaps.getBitmap(bitmaps.getIndex(value))); }
private static long computeTotalCard(final QueryableIndex index, final Iterable<DimensionSpec> dimensionSpecs) { long totalCard = 0; for (DimensionSpec dimension : dimensionSpecs) { final ColumnHolder columnHolder = index.getColumnHolder(dimension.getDimension()); if (columnHolder != null) { final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); if (bitmapIndex != null) { totalCard += bitmapIndex.getCardinality(); } } } return totalCard; } }
@Override @Nullable public Comparable getMinValue(String dimension) { ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null; } return null; }
@Override public BitmapValues getBitmapValues(String dimension, int dictId) { final ColumnHolder columnHolder = input.getColumnHolder(dimension); if (columnHolder == null) { return BitmapValues.EMPTY; } final BitmapIndex bitmaps = columnHolder.getBitmapIndex(); if (bitmaps == null) { return BitmapValues.EMPTY; } if (dictId >= 0) { return new ImmutableBitmapValues(bitmaps.getBitmap(dictId)); } else { return BitmapValues.EMPTY; } }
@Override public DateTime getMaxTime() { try (final NumericColumn column = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return DateTimes.utc(column.getLongSingleValueRow(column.length() - 1)); } }
@Override public String getColumnTypeName(String columnName) { final ColumnHolder columnHolder = index.getColumnHolder(columnName); try (final BaseColumn col = columnHolder.getColumn()) { if (col instanceof ComplexColumn) { return ((ComplexColumn) col).getTypeName(); } else { return columnHolder.getCapabilities().getType().toString(); } } catch (IOException e) { throw new UncheckedIOException(e); } }
@Override @Nullable public Comparable getMaxValue(String dimension) { ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null; } return null; }
private void assertDimCompression(QueryableIndex index, CompressionStrategy expectedStrategy) throws Exception { // Java voodoo if (expectedStrategy == null || expectedStrategy == CompressionStrategy.UNCOMPRESSED) { return; } DictionaryEncodedColumn encodedColumn = (DictionaryEncodedColumn) index.getColumnHolder("dim2").getColumn(); Object obj; if (encodedColumn.hasMultipleValues()) { Field field = StringDictionaryEncodedColumn.class.getDeclaredField("multiValueColumn"); field.setAccessible(true); obj = field.get(encodedColumn); } else { Field field = StringDictionaryEncodedColumn.class.getDeclaredField("column"); field.setAccessible(true); obj = field.get(encodedColumn); } // CompressedVSizeColumnarIntsSupplier$CompressedByteSizeColumnarInts // CompressedVSizeColumnarMultiIntsSupplier$CompressedVSizeColumnarMultiInts Field compressedSupplierField = obj.getClass().getDeclaredField("this$0"); compressedSupplierField.setAccessible(true); Object supplier = compressedSupplierField.get(obj); Field compressionField = supplier.getClass().getDeclaredField("compression"); compressionField.setAccessible(true); Object strategy = compressionField.get(supplier); Assert.assertEquals(expectedStrategy, strategy); }
@Override public ColumnValueSelector<?> makeColumnValueSelector(String columnName) { if (virtualColumns.exists(columnName)) { return virtualColumns.makeColumnValueSelector(columnName, this); } BaseColumn column = columnCache.computeIfAbsent(columnName, name -> { ColumnHolder holder = index.getColumnHolder(name); if (holder != null) { return closer.register(holder.getColumn()); } else { return null; } }); if (column != null) { return column.makeColumnValueSelector(offset); } else { return NilColumnValueSelector.instance(); } }
@Override public String getMetricType(String metric) { final ColumnHolder columnHolder = input.getColumnHolder(metric); final ValueType type = columnHolder.getCapabilities().getType(); switch (type) { case FLOAT: return "float"; case LONG: return "long"; case DOUBLE: return "double"; case COMPLEX: { try (ComplexColumn complexColumn = (ComplexColumn) columnHolder.getColumn()) { return complexColumn.getTypeName(); } } default: throw new ISE("Unknown type[%s]", type); } }
final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); final ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null) { return DimensionSelector.constant(null, extractionFn);
final ColumnHolder columnHolder = index.getColumnHolder(dimension.getDimension()); if (columnHolder == null) { continue;
); Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size());
@Test public void testPersistWithSegmentMetadata() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, toPersist); Map<String, Object> metadataElems = ImmutableMap.of("key", "value"); toPersist.getMetadata().putAll(metadataElems); final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression()); Assert.assertEquals( new Metadata( metadataElems, IncrementalIndexTest.getDefaultCombiningAggregatorFactories(), null, Granularities.NONE, Boolean.TRUE ), index.getMetadata() ); }
Assert.assertEquals(3, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals( Arrays.asList("dim1", "dim2"),
@Test public void testPersist() throws Exception { final long timestamp = System.currentTimeMillis(); IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null); IncrementalIndexTest.populateIndex(timestamp, toPersist); final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression()); Assert.assertArrayEquals( IncrementalIndexTest.getDefaultCombiningAggregatorFactories(), index.getMetadata().getAggregators() ); Assert.assertEquals( Granularities.NONE, index.getMetadata().getQueryGranularity() ); }