@Override public IntSet deserialize(ByteBuffer byteBuffer) { int size = byteBuffer.getInt(); IntSet intSet = new IntOpenHashSet(size); for (int i = 0; i < size; i++) { intSet.add(byteBuffer.getInt()); } return intSet; } };
/** * Collect statistics for the given entry. * - Add it to the passed in set (which could be raw or aggregated) * - Update maximum number of values for Multi-valued entries * - Update Total number of entries * - Check if entry is sorted. * @param entry * @param set */ private void collectEntry(Object entry, IntSet set) { if (entry instanceof Object[]) { for (Object e : (Object[]) entry) { set.add(((Number) e).intValue()); } if (maxNumberOfMultiValues < ((Object[]) entry).length) { maxNumberOfMultiValues = ((Object[]) entry).length; } updateTotalNumberOfEntries((Object[]) entry); } else { int value = ((Number) entry).intValue(); addressSorted(value); updatePartition(value); set.add(value); totalNumberOfEntries++; } }
IntRawValueBasedNotInPredicateEvaluator(NotInPredicate notInPredicate) { String[] values = notInPredicate.getValues(); _nonMatchingValues = new IntOpenHashSet(HashUtil.getMinHashSetSize(values.length)); for (String value : values) { _nonMatchingValues.add(Integer.parseInt(value)); } }
IntRawValueBasedInPredicateEvaluator(InPredicate inPredicate) { String[] values = inPredicate.getValues(); _matchingValues = new IntOpenHashSet(HashUtil.getMinHashSetSize(values.length)); for (String value : values) { _matchingValues.add(Integer.parseInt(value)); } }
@Test public void testIntSet() { for (int i = 0; i < NUM_ITERATIONS; i++) { int size = RANDOM.nextInt(100); IntSet expected = new IntOpenHashSet(size); for (int j = 0; j < size; j++) { expected.add(RANDOM.nextInt()); } byte[] bytes = ObjectSerDeUtils.serialize(expected); IntSet actual = ObjectSerDeUtils.deserialize(bytes, ObjectSerDeUtils.ObjectType.IntSet); // NOTE: use Object comparison instead of Collection comparison because the order might be different assertEquals((Object) actual, expected, ERROR_MESSAGE); } }
@Override public int[][] transformToIntValuesMV(@Nonnull ProjectionBlock projectionBlock) { if (getResultMetadata().getDataType() != FieldSpec.DataType.INT) { return super.transformToIntValuesMV(projectionBlock); } if (_intValueSet == null) { _intValueSet = new IntOpenHashSet(); for (String inValue : _stringValueSet) { _intValueSet.add(Integer.parseInt(inValue)); } _intValues = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; } int[][] unFilteredIntValues = _mainTransformFunction.transformToIntValuesMV(projectionBlock); int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _intValues[i] = filterInts(_intValueSet, unFilteredIntValues[i]); } return _intValues; }
@Override public int countUnique() { IntSet ints = new IntOpenHashSet(size()); for (int i = 0; i < size(); i++) { ints.add(data.getInt(i)); } return ints.size(); }
@Override public int[][] transformToDictIdsMV(@Nonnull ProjectionBlock projectionBlock) { if (_dictIdSet == null) { _dictIdSet = new IntOpenHashSet(); Dictionary dictionary = _mainTransformFunction.getDictionary(); for (String inValue : _stringValueSet) { int dictId = dictionary.indexOf(inValue); if (dictId >= 0) { _dictIdSet.add(dictId); } } _dictIds = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; } int[][] unFilteredDictIds = _mainTransformFunction.transformToDictIdsMV(projectionBlock); int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _dictIds[i] = filterInts(_dictIdSet, unFilteredDictIds[i]); } return _dictIds; }
RealtimeDictionaryBasedRangePredicateEvaluator(RangePredicate rangePredicate, MutableDictionary dictionary) { _matchingDictIdSet = new IntOpenHashSet(); int dictionarySize = dictionary.length(); if (dictionarySize == 0) { _numMatchingDictIds = 0; _alwaysFalse = true; return; } String lowerBoundary = rangePredicate.getLowerBoundary(); String upperBoundary = rangePredicate.getUpperBoundary(); boolean includeLowerBoundary = rangePredicate.includeLowerBoundary(); boolean includeUpperBoundary = rangePredicate.includeUpperBoundary(); if (lowerBoundary.equals("*")) { lowerBoundary = dictionary.getMinVal().toString(); } if (upperBoundary.equals("*")) { upperBoundary = dictionary.getMaxVal().toString(); } for (int dictId = 0; dictId < dictionarySize; dictId++) { if (dictionary.inRange(lowerBoundary, upperBoundary, dictId, includeLowerBoundary, includeUpperBoundary)) { _matchingDictIdSet.add(dictId); } } _numMatchingDictIds = _matchingDictIdSet.size(); if (_numMatchingDictIds == 0) { _alwaysFalse = true; } else if (dictionarySize == _numMatchingDictIds) { _alwaysTrue = true; } }
@Override public int countUnique() { IntSet uniqueElements = new IntOpenHashSet(); for (int i = 0; i < size(); i++) { if (!isMissingValue(getInt(i))) { uniqueElements.add(getInt(i)); } } return uniqueElements.size(); }
PredicateEvaluator firstPredicateEvaluator = predicateEvaluators.get(0); for (int matchingDictId : firstPredicateEvaluator.getMatchingDictIds()) { matchingDictIds.add(matchingDictId);
DictionaryBasedInPredicateEvaluator(InPredicate inPredicate, Dictionary dictionary) { String[] values = inPredicate.getValues(); _matchingDictIdSet = new IntOpenHashSet(HashUtil.getMinHashSetSize(values.length)); for (String value : values) { int dictId = dictionary.indexOf(value); if (dictId >= 0) { _matchingDictIdSet.add(dictId); } } _numMatchingDictIds = _matchingDictIdSet.size(); if (_numMatchingDictIds == 0) { _alwaysFalse = true; } else if (dictionary.length() == _numMatchingDictIds) { _alwaysTrue = true; } }
DictionaryBasedNotInPredicateEvaluator(NotInPredicate notInPredicate, Dictionary dictionary) { String[] values = notInPredicate.getValues(); _nonMatchingDictIdSet = new IntOpenHashSet(HashUtil.getMinHashSetSize(values.length)); for (String value : values) { int dictId = dictionary.indexOf(value); if (dictId >= 0) { _nonMatchingDictIdSet.add(dictId); } } _numNonMatchingDictIds = _nonMatchingDictIdSet.size(); if (_numNonMatchingDictIds == 0) { _alwaysTrue = true; } else if (dictionary.length() == _numNonMatchingDictIds) { _alwaysFalse = true; } _dictionary = dictionary; }
@Override public IntColumn unique() { final IntSet values = new IntOpenHashSet(); for (int i = 0; i < size(); i++) { if (!isMissing(i)) { values.add(getInt(i)); } } final IntColumn column = IntColumn.create(name() + " Unique values"); for (int value : values) { column.append(value); } return column; }
int value = _random.nextInt(); stringValues.add(Integer.toString(value)); valueSet.add(value);
private static void testFilter(DictionaryAwarePageFilter filter, Block block, boolean filterRange) { IntSet actualSelectedPositions = toSet(filter.filter(null, new Page(block))); block = block.getLoadedBlock(); IntSet expectedSelectedPositions = new IntArraySet(block.getPositionCount()); for (int position = 0; position < block.getPositionCount(); position++) { if (isSelected(filterRange, block.getLong(position, 0))) { expectedSelectedPositions.add(position); } } assertEquals(actualSelectedPositions, expectedSelectedPositions); }
@Override public DateColumn unique() { IntSet ints = new IntOpenHashSet(data.size()); for (int i = 0; i < size(); i++) { ints.add(data.getInt(i)); } DateColumn copy = emptyCopy(ints.size()); copy.setName(name() + " Unique values"); copy.data = IntArrayList.wrap(ints.toIntArray()); return copy; }
private int getResults( final DocumentIterator documentIterator, final int offset, final int length, final ObjectArrayList<DocumentScoreInfo<Reference2ObjectMap<Index,SelectedInterval[]>>> results, final IntSet alreadySeen ) throws IOException { int document, count = 0; // Number of not-already-seen documents // Unfortunately, to provide the exact count of results we have to scan the whole iterator. while ( ( document = documentIterator.nextDocument() ) != -1 ) { if ( alreadySeen != null && ! alreadySeen.add( document ) ) continue; if ( count >= offset && count < offset + length ) results.add( new DocumentScoreInfo<Reference2ObjectMap<Index,SelectedInterval[]>>( document, -1 ) ); count++; } return count; }
@Override public int countUnique() { IntSet ints = new IntOpenHashSet(size()); for (int i = 0; i < size(); i++) { ints.add(data.getInt(i)); } return ints.size(); }