private ColumnChunk readMap(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %d", parameters.size()); Block[] blocks = new Block[parameters.size()]; ColumnChunk columnChunk = readColumnChunk(field.getChildren().get(0).get()); blocks[0] = columnChunk.getBlock(); blocks[1] = readColumnChunk(field.getChildren().get(1).get()).getBlock(); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]); return new ColumnChunk(mapBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
private ColumnChunk readArray(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %d", parameters.size()); Field elementField = field.getChildren().get(0).get(); ColumnChunk columnChunk = readColumnChunk(elementField); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock()); return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
public ColumnChunk readPrimitive(Field field) throws IOException { IntList definitionLevels = new IntArrayList(); IntList repetitionLevels = new IntArrayList(); seek(); BlockBuilder blockBuilder = field.getType().createBlockBuilder(null, nextBatchSize); int valueCount = 0; while (valueCount < nextBatchSize) { if (page == null) { readNextPage(); } int valuesToRead = Math.min(remainingValueCountInPage, nextBatchSize - valueCount); readValues(blockBuilder, valuesToRead, field.getType(), definitionLevels, repetitionLevels); valueCount += valuesToRead; } checkArgument(valueCount == nextBatchSize, "valueCount %s not equals to batchSize %s", valueCount, nextBatchSize); readOffset = 0; nextBatchSize = 0; return new ColumnChunk(blockBuilder.build(), definitionLevels.toIntArray(), repetitionLevels.toIntArray()); }
private static int[] filterInts(IntSet intSet, int[] source) { IntList intList = new IntArrayList(); for (int value : source) { if (intSet.contains(value)) { intList.add(value); } } if (intList.size() == source.length) { return source; } else { return intList.toIntArray(); } }
@Override public int[] getMatchingDictIds() { if (_matchingDictIds == null) { IntList matchingDictIds = new IntArrayList(); int dictionarySize = _dictionary.length(); for (int dictId = 0; dictId < dictionarySize; dictId++) { if (applySV(dictId)) { matchingDictIds.add(dictId); } } _matchingDictIds = matchingDictIds.toIntArray(); } return _matchingDictIds; } }
int[] expectedValues = expectedList.toIntArray();
@Override public int[] toIntArray() { return list.toIntArray(); } @Override
@Override public double scoreNGrams(IntList document, int W) { return score(SparseIntArray.histogram(document.toIntArray(), W)); } }
@Override public double scoreNGrams(IntList document, int W) { return score(SparseIntArray.histogram(document.toIntArray(), W)); }
@SuppressWarnings("unchecked") @Override public TypeComparator<Record> createComparator() { final TypeConfig<Record> typeConfig = this.typeConfigReference[0]; final RecordKeyExtractor keyExtractor = (RecordKeyExtractor) typeConfig.getKeyExtractor(); return new RecordComparator(keyExtractor.getIndices().toIntArray(), keyExtractor.getKeyClasses().toArray(new Class[0])); }
public static int[] convertIndexes(FacetDataCache<?> dataCache, String[] vals) { IntList list = new IntArrayList(); for (String val : vals) { int[] range = parse(dataCache, val); if (range != null) { for (int i = range[0]; i <= range[1]; ++i) { list.add(i); } } } return list.toIntArray(); }
private static int[] convertString(FacetDataCache<?> dataCache, String[] vals) { IntList list = new IntArrayList(vals.length); for (int i = 0; i < vals.length; ++i) { int index = dataCache.valArray.indexOf(vals[i]); if (index >= 0) { list.add(index); } } return list.toIntArray(); }
public ColumnChunk readPrimitive(Field field) throws IOException { IntList definitionLevels = new IntArrayList(); IntList repetitionLevels = new IntArrayList(); seek(); BlockBuilder blockBuilder = field.getType().createBlockBuilder(null, nextBatchSize); int valueCount = 0; while (valueCount < nextBatchSize) { if (page == null) { readNextPage(); } int valuesToRead = Math.min(remainingValueCountInPage, nextBatchSize - valueCount); readValues(blockBuilder, valuesToRead, field.getType(), definitionLevels, repetitionLevels); valueCount += valuesToRead; } checkArgument(valueCount == nextBatchSize, "valueCount %s not equals to batchSize %s", valueCount, nextBatchSize); readOffset = 0; nextBatchSize = 0; return new ColumnChunk(blockBuilder.build(), definitionLevels.toIntArray(), repetitionLevels.toIntArray()); }
private ColumnChunk readMap(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %d", parameters.size()); Block[] blocks = new Block[parameters.size()]; ColumnChunk columnChunk = readColumnChunk(field.getChildren().get(0).get()); blocks[0] = columnChunk.getBlock(); blocks[1] = readColumnChunk(field.getChildren().get(1).get()).getBlock(); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]); return new ColumnChunk(mapBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
private ColumnChunk readMap(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %d", parameters.size()); Block[] blocks = new Block[parameters.size()]; ColumnChunk columnChunk = readColumnChunk(field.getChildren().get(0).get()); blocks[0] = columnChunk.getBlock(); blocks[1] = readColumnChunk(field.getChildren().get(1).get()).getBlock(); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]); return new ColumnChunk(mapBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
private ColumnChunk readArray(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %d", parameters.size()); Field elementField = field.getChildren().get(0).get(); ColumnChunk columnChunk = readColumnChunk(elementField); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock()); return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
public ColumnChunk readPrimitive(Field field) throws IOException { IntList definitionLevels = new IntArrayList(); IntList repetitionLevels = new IntArrayList(); seek(); BlockBuilder blockBuilder = field.getType().createBlockBuilder(null, nextBatchSize); int valueCount = 0; while (valueCount < nextBatchSize) { if (page == null) { readNextPage(); } int valuesToRead = Math.min(remainingValueCountInPage, nextBatchSize - valueCount); readValues(blockBuilder, valuesToRead, field.getType(), definitionLevels, repetitionLevels); valueCount += valuesToRead; } checkArgument(valueCount == nextBatchSize, "valueCount %s not equals to batchSize %s", valueCount, nextBatchSize); readOffset = 0; nextBatchSize = 0; return new ColumnChunk(blockBuilder.build(), definitionLevels.toIntArray(), repetitionLevels.toIntArray()); }
private ColumnChunk readArray(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %d", parameters.size()); Field elementField = field.getChildren().get(0).get(); ColumnChunk columnChunk = readColumnChunk(elementField); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock()); return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
/** * Same as convert(FacetDataCache dataCache,String[] vals) except that the * values are supplied in raw form so that we can take advantage of the type * information to find index faster. * * @param <T> * @param dataCache * @param vals * @return the array of order indices of the values. */ public static <T> int[] convert(FacetDataCache<T> dataCache, T[] vals) { if (vals != null && (vals instanceof String[])) return convertString(dataCache, (String[]) vals); IntList list = new IntArrayList(vals.length); for (int i = 0; i < vals.length; ++i) { int index = dataCache.valArray.indexOfWithType(vals[i]); if (index >= 0) { list.add(index); } } return list.toIntArray(); }
static OffsetIndex filterOffsetIndex(OffsetIndex offsetIndex, RowRanges rowRanges, long totalRowCount) { IntList indexMap = new IntArrayList(); for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) { long from = offsetIndex.getFirstRowIndex(i); if (rowRanges.isOverlapping(from, offsetIndex.getLastRowIndex(i, totalRowCount))) { indexMap.add(i); } } return new FilteredOffsetIndex(offsetIndex, indexMap.toIntArray()); }