public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
public void addValue(int docID, BytesRef value) { if (docID < pending.size()) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } if (value == null) { throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": null value not allowed"); } if (value.length > (BYTE_BLOCK_SIZE - 2)) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" is too large, must be <= " + (BYTE_BLOCK_SIZE - 2)); } // Fill in any holes: while(pending.size() < docID) { pending.add(EMPTY_ORD); } addOneValue(value); }
public void addValue(int docID, BytesRef value) { if (docID < pending.size()) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } if (value == null) { throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": null value not allowed"); } if (value.length > (BYTE_BLOCK_SIZE - 2)) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" is too large, must be <= " + (BYTE_BLOCK_SIZE - 2)); } // Fill in any holes: while(pending.size() < docID) { pending.add(EMPTY_ORD); } addOneValue(value); }
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) { final long termOrd = builder.nextOrdinal(); assert termOrd == termOrdToBytesOffset.size(); termOrdToBytesOffset.add(bytes.copyUsingLengthPrefix(term)); docsEnum = termsEnum.postings(docsEnum, PostingsEnum.NONE);
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) { final long termOrd = builder.nextOrdinal(); assert termOrd == termOrdToBytesOffset.size(); termOrdToBytesOffset.add(bytes.copyUsingLengthPrefix(term)); docsEnum = termsEnum.postings(docsEnum, PostingsEnum.NONE);
final int maxDoc = state.segmentInfo.maxDoc(); final int maxCountPerDoc = maxCount; assert pendingCounts.size() == maxDoc; final int valueCount = hash.size(); final PackedLongValues ords = pending.build();
final int maxDoc = state.segmentInfo.maxDoc(); final int maxCountPerDoc = maxCount; assert pendingCounts.size() == maxDoc; final int valueCount = hash.size(); final PackedLongValues ords = pending.build();
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pending.size() == maxDoc; final int valueCount = hash.size(); final PackedLongValues ords = pending.build(); final int[] sortedValues = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator()); final int[] ordMap = new int[valueCount]; for(int ord=0;ord<valueCount;ord++) { ordMap[sortedValues[ord]] = ord; } dvConsumer.addSortedField(fieldInfo, // ord -> value new Iterable<BytesRef>() { @Override public Iterator<BytesRef> iterator() { return new ValuesIterator(sortedValues, valueCount, hash); } }, // doc -> ord new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new OrdsIterator(ordMap, maxDoc, ords); } }); }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pending.size() == maxDoc; final int valueCount = hash.size(); final PackedLongValues ords = pending.build(); final int[] sortedValues = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator()); final int[] ordMap = new int[valueCount]; for(int ord=0;ord<valueCount;ord++) { ordMap[sortedValues[ord]] = ord; } dvConsumer.addSortedField(fieldInfo, // ord -> value new Iterable<BytesRef>() { @Override public Iterator<BytesRef> iterator() { return new ValuesIterator(sortedValues, valueCount, hash); } }, // doc -> ord new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new OrdsIterator(ordMap, maxDoc, ords); } }); }
public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pendingCounts.size() == maxDoc; final PackedLongValues values = pending.build(); final PackedLongValues valueCounts = pendingCounts.build(); dvConsumer.addSortedNumericField(fieldInfo, // doc -> valueCount new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new CountIterator(valueCounts); } }, // values new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new ValuesIterator(values); } }); }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pendingCounts.size() == maxDoc; final PackedLongValues values = pending.build(); final PackedLongValues valueCounts = pendingCounts.build(); dvConsumer.addSortedNumericField(fieldInfo, // doc -> valueCount new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new CountIterator(valueCounts); } }, // values new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new ValuesIterator(values); } }); }
public void addValue(int docID, long value) { if (docID < pending.size()) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } // Fill in any holes: for (int i = (int)pending.size(); i < docID; ++i) { pending.add(MISSING); } pending.add(value); docsWithField = FixedBitSet.ensureCapacity(docsWithField, docID); docsWithField.set(docID); updateBytesUsed(); }
public void addValue(int docID, long value) { if (docID < pending.size()) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } // Fill in any holes: for (int i = (int)pending.size(); i < docID; ++i) { pending.add(MISSING); } pending.add(value); docsWithField = FixedBitSet.ensureCapacity(docsWithField, docID); docsWithField.set(docID); updateBytesUsed(); }
@Override public void finish(int maxDoc) { while(pending.size() < maxDoc) { pending.add(EMPTY_ORD); } updateBytesUsed(); }
public void addValue(int docID, long value) { // Fill in any holes: for (int i = (int)pending.size(); i < docID; ++i) { pending.add(MISSING); } pending.add(value); updateBytesUsed(); }
public void addValue(int docID, long value) { // Fill in any holes: for (int i = (int)pending.size(); i < docID; ++i) { pending.add(MISSING); } pending.add(value); updateBytesUsed(); }
@Override public void finish(int maxDoc) { while(pending.size() < maxDoc) { pending.add(EMPTY_ORD); } updateBytesUsed(); }
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) { final long termOrd = builder.nextOrdinal(); assert termOrd == termOrdToBytesOffset.size(); termOrdToBytesOffset.add(bytes.copyUsingLengthPrefix(term)); docsEnum = termsEnum.postings(docsEnum, PostingsEnum.NONE);