@Override public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED_NUMERIC); long[] stats = writeValues(field, valuesProducer); int numDocsWithField = Math.toIntExact(stats[0]); long numValues = stats[1]; assert numValues >= numDocsWithField; meta.writeInt(numDocsWithField); if (numValues > numDocsWithField) { long start = data.getFilePointer(); meta.writeLong(start); meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT); long addr = 0; addressesWriter.add(addr); SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { addr += values.docValueCount(); addressesWriter.add(addr); } addressesWriter.finish(); meta.writeLong(data.getFilePointer() - start); } }
/** Returns an instance suitable for encoding {@code numValues} into monotonic * blocks of 2<sup>{@code blockShift}</sup> values. Metadata will be written * to {@code metaOut} and actual data to {@code dataOut}. */ public static DirectMonotonicWriter getInstance(IndexOutput metaOut, IndexOutput dataOut, long numValues, int blockShift) { return new DirectMonotonicWriter(metaOut, dataOut, numValues, blockShift); }
/** This must be called exactly once after all values have been {@link #add(long) added}. */ public void finish() throws IOException { if (count != numValues) { throw new IllegalStateException("Wrong number of values added, expected: " + numValues + ", got: " + count); } if (finished) { throw new IllegalStateException("#finish has been called already"); } if (bufferSize > 0) { flush(); } finished = true; }
DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, addressBuffer, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { if ((ord & Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) { writer.add(offset); final int sortKeyLength; if (ord == 0) { writer.add(offset); writer.finish(); meta.writeLong(start); meta.writeLong(data.getFilePointer() - start);
/** Write a new value. Note that data might not make it to storage until * {@link #finish()} is called. * @throws IllegalArgumentException if values don't come in order */ public void add(long v) throws IOException { if (v < previous) { throw new IllegalArgumentException("Values do not come in order: " + previous + ", " + v); } if (bufferSize == buffer.length) { flush(); } buffer[bufferSize++] = v; previous = v; count++; }
/** Returns an instance suitable for encoding {@code numValues} into monotonic * blocks of 2<sup>{@code blockShift}</sup> values. Metadata will be written * to {@code metaOut} and actual data to {@code dataOut}. */ public static DirectMonotonicWriter getInstance(IndexOutput metaOut, IndexOutput dataOut, long numValues, int blockShift) { return new DirectMonotonicWriter(metaOut, dataOut, numValues, blockShift); }
meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT); long numBlocks = (size + Lucene70DocValuesFormat.TERMS_DICT_BLOCK_MASK) >>> Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT; DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, addressBuffer, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { if ((ord & Lucene70DocValuesFormat.TERMS_DICT_BLOCK_MASK) == 0) { writer.add(data.getFilePointer() - start); data.writeVInt(term.length); data.writeBytes(term.bytes, term.offset, term.length); ++ord; writer.finish(); meta.writeInt(maxLength); meta.writeLong(start);
/** Write a new value. Note that data might not make it to storage until * {@link #finish()} is called. * @throws IllegalArgumentException if values don't come in order */ public void add(long v) throws IOException { if (v < previous) { throw new IllegalArgumentException("Values do not come in order: " + previous + ", " + v); } if (bufferSize == buffer.length) { flush(); } buffer[bufferSize++] = v; previous = v; count++; }
/** Returns an instance suitable for encoding {@code numValues} into monotonic * blocks of 2<sup>{@code blockShift}</sup> values. Metadata will be written * to {@code metaOut} and actual data to {@code dataOut}. */ public static DirectMonotonicWriter getInstance(IndexOutput metaOut, IndexOutput dataOut, long numValues, int blockShift) { return new DirectMonotonicWriter(metaOut, dataOut, numValues, blockShift); }
meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1, DIRECT_MONOTONIC_BLOCK_SHIFT); long addr = 0; writer.add(addr); values = valuesProducer.getBinary(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { addr += values.binaryValue().length; writer.add(addr); writer.finish(); meta.writeLong(data.getFilePointer() - start);
/** Write a new value. Note that data might not make it to storage until * {@link #finish()} is called. * @throws IllegalArgumentException if values don't come in order */ public void add(long v) throws IOException { if (v < previous) { throw new IllegalArgumentException("Values do not come in order: " + previous + ", " + v); } if (bufferSize == buffer.length) { flush(); } buffer[bufferSize++] = v; previous = v; count++; }
/** Returns an instance suitable for encoding {@code numValues} into monotonic * blocks of 2<sup>{@code blockShift}</sup> values. Metadata will be written * to {@code metaOut} and actual data to {@code dataOut}. */ public static DirectMonotonicWriter getInstance(IndexOutput metaOut, IndexOutput dataOut, long numValues, int blockShift) { return new DirectMonotonicWriter(metaOut, dataOut, numValues, blockShift); }
meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1, DIRECT_MONOTONIC_BLOCK_SHIFT); long addr = 0; addressesWriter.add(addr); values = valuesProducer.getSortedSet(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { addr++; addressesWriter.add(addr); addressesWriter.finish(); meta.writeLong(data.getFilePointer() - start);
/** Write a new value. Note that data might not make it to storage until * {@link #finish()} is called. * @throws IllegalArgumentException if values don't come in order */ public void add(long v) throws IOException { if (v < previous) { throw new IllegalArgumentException("Values do not come in order: " + previous + ", " + v); } if (bufferSize == buffer.length) { flush(); } buffer[bufferSize++] = v; previous = v; count++; }
long writeSparseMissingBitset(Iterable<Number> values, NumberType numberType, long numDocsWithValue) throws IOException { meta.writeVLong(numDocsWithValue); // Write doc IDs that have a value meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter docIdsWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithValue, DIRECT_MONOTONIC_BLOCK_SHIFT); long docID = 0; for (Number nv : values) { switch (numberType) { case VALUE: if (nv != null) { docIdsWriter.add(docID); } break; case ORDINAL: if (nv.longValue() != -1L) { docIdsWriter.add(docID); } break; default: throw new AssertionError(); } docID++; } docIdsWriter.finish(); return docID; }
/** This must be called exactly once after all values have been {@link #add(long) added}. */ public void finish() throws IOException { if (count != numValues) { throw new IllegalStateException("Wrong number of values added, expected: " + numValues + ", got: " + count); } if (finished) { throw new IllegalStateException("#finish has been called already"); } if (bufferSize > 0) { flush(); } finished = true; }
long writeSparseMissingBitset(Iterable<Number> values, NumberType numberType, long numDocsWithValue) throws IOException { meta.writeVLong(numDocsWithValue); // Write doc IDs that have a value meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter docIdsWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithValue, DIRECT_MONOTONIC_BLOCK_SHIFT); long docID = 0; for (Number nv : values) { switch (numberType) { case VALUE: if (nv != null) { docIdsWriter.add(docID); } break; case ORDINAL: if (nv.longValue() != -1L) { docIdsWriter.add(docID); } break; default: throw new AssertionError(); } docID++; } docIdsWriter.finish(); return docID; }
/** This must be called exactly once after all values have been {@link #add(long) added}. */ public void finish() throws IOException { if (count != numValues) { throw new IllegalStateException("Wrong number of values added, expected: " + numValues + ", got: " + count); } if (finished) { throw new IllegalStateException("#finish has been called already"); } if (bufferSize > 0) { flush(); } finished = true; }
long writeSparseMissingBitset(Iterable<Number> values, NumberType numberType, long numDocsWithValue) throws IOException { meta.writeVLong(numDocsWithValue); // Write doc IDs that have a value meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter docIdsWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithValue, DIRECT_MONOTONIC_BLOCK_SHIFT); long docID = 0; for (Number nv : values) { switch (numberType) { case VALUE: if (nv != null) { docIdsWriter.add(docID); } break; case ORDINAL: if (nv.longValue() != -1L) { docIdsWriter.add(docID); } break; default: throw new AssertionError(); } docID++; } docIdsWriter.finish(); return docID; }
/** This must be called exactly once after all values have been {@link #add(long) added}. */ public void finish() throws IOException { if (count != numValues) { throw new IllegalStateException("Wrong number of values added, expected: " + numValues + ", got: " + count); } if (finished) { throw new IllegalStateException("#finish has been called already"); } if (bufferSize > 0) { flush(); } finished = true; }