/** Returns a sorted array containing unique field numbers */ private int[] flushFieldNums() throws IOException { SortedSet<Integer> fieldNums = new TreeSet<>(); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { fieldNums.add(fd.fieldNum); } } final int numDistinctFields = fieldNums.size(); assert numDistinctFields > 0; final int bitsRequired = PackedInts.bitsRequired(fieldNums.last()); final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired; vectorsStream.writeByte((byte) token); if (numDistinctFields - 1 >= 0x07) { vectorsStream.writeVInt(numDistinctFields - 1 - 0x07); } final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1); for (Integer fieldNum : fieldNums) { writer.add(fieldNum); } writer.finish(); int[] fns = new int[fieldNums.size()]; int i = 0; for (Integer key : fieldNums) { fns[i++] = key; } return fns; }
private void flushFields(int totalFields, int[] fieldNums) throws IOException { final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, PackedInts.bitsRequired(fieldNums.length - 1), 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum); assert fieldNumIndex >= 0; writer.add(fieldNumIndex); } } writer.finish(); }
private static void saveInts(int[] values, int length, DataOutput out) throws IOException { assert length > 0; if (length == 1) { out.writeVInt(values[0]); } else { boolean allEqual = true; for (int i = 1; i < length; ++i) { if (values[i] != values[0]) { allEqual = false; break; } } if (allEqual) { out.writeVInt(0); out.writeVInt(values[0]); } else { long max = 0; for (int i = 0; i < length; ++i) { max |= values[i]; } final int bitsRequired = PackedInts.bitsRequired(max); out.writeVInt(bitsRequired); final PackedInts.Writer w = PackedInts.getWriterNoHeader(out, PackedInts.Format.PACKED, length, bitsRequired, 1); for (int i = 0; i < length; ++i) { w.add(values[i]); } w.finish(); } } }
/** Returns a sorted array containing unique field numbers */ private int[] flushFieldNums() throws IOException { SortedSet<Integer> fieldNums = new TreeSet<>(); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { fieldNums.add(fd.fieldNum); } } final int numDistinctFields = fieldNums.size(); assert numDistinctFields > 0; final int bitsRequired = PackedInts.bitsRequired(fieldNums.last()); final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired; vectorsStream.writeByte((byte) token); if (numDistinctFields - 1 >= 0x07) { vectorsStream.writeVInt(numDistinctFields - 1 - 0x07); } final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1); for (Integer fieldNum : fieldNums) { writer.add(fieldNum); } writer.finish(); int[] fns = new int[fieldNums.size()]; int i = 0; for (Integer key : fieldNums) { fns[i++] = key; } return fns; }
/** Returns a sorted array containing unique field numbers */ private int[] flushFieldNums() throws IOException { SortedSet<Integer> fieldNums = new TreeSet<>(); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { fieldNums.add(fd.fieldNum); } } final int numDistinctFields = fieldNums.size(); assert numDistinctFields > 0; final int bitsRequired = PackedInts.bitsRequired(fieldNums.last()); final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired; vectorsStream.writeByte((byte) token); if (numDistinctFields - 1 >= 0x07) { vectorsStream.writeVInt(numDistinctFields - 1 - 0x07); } final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1); for (Integer fieldNum : fieldNums) { writer.add(fieldNum); } writer.finish(); int[] fns = new int[fieldNums.size()]; int i = 0; for (Integer key : fieldNums) { fns[i++] = key; } return fns; }
private static void saveInts(int[] values, int length, DataOutput out) throws IOException { assert length > 0; if (length == 1) { out.writeVInt(values[0]); } else { boolean allEqual = true; for (int i = 1; i < length; ++i) { if (values[i] != values[0]) { allEqual = false; break; } } if (allEqual) { out.writeVInt(0); out.writeVInt(values[0]); } else { long max = 0; for (int i = 0; i < length; ++i) { max |= values[i]; } final int bitsRequired = PackedInts.bitsRequired(max); out.writeVInt(bitsRequired); final PackedInts.Writer w = PackedInts.getWriterNoHeader(out, PackedInts.Format.PACKED, length, bitsRequired, 1); for (int i = 0; i < length; ++i) { w.add(values[i]); } w.finish(); } } }
private void flushNumTerms(int totalFields) throws IOException { int maxNumTerms = 0; for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { maxNumTerms |= fd.numTerms; } } final int bitsRequired = PackedInts.bitsRequired(maxNumTerms); vectorsStream.writeVInt(bitsRequired); final PackedInts.Writer writer = PackedInts.getWriterNoHeader( vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { writer.add(fd.numTerms); } } assert writer.ord() == totalFields - 1; writer.finish(); }
private void flushNumTerms(int totalFields) throws IOException { int maxNumTerms = 0; for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { maxNumTerms |= fd.numTerms; } } final int bitsRequired = PackedInts.bitsRequired(maxNumTerms); vectorsStream.writeVInt(bitsRequired); final PackedInts.Writer writer = PackedInts.getWriterNoHeader( vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { writer.add(fd.numTerms); } } assert writer.ord() == totalFields - 1; writer.finish(); }
private void flushNumTerms(int totalFields) throws IOException { int maxNumTerms = 0; for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { maxNumTerms |= fd.numTerms; } } final int bitsRequired = PackedInts.bitsRequired(maxNumTerms); vectorsStream.writeVInt(bitsRequired); final PackedInts.Writer writer = PackedInts.getWriterNoHeader( vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { writer.add(fd.numTerms); } } assert writer.ord() == totalFields - 1; writer.finish(); }
/** * Save this mutable into <code>out</code>. Instantiating a reader from * the generated data will return a reader with the same number of bits * per value. */ public void save(DataOutput out) throws IOException { Writer writer = getWriterNoHeader(out, getFormat(), size(), getBitsPerValue(), DEFAULT_BUFFER_SIZE); writer.writeHeader(); for (int i = 0; i < size(); ++i) { writer.add(get(i)); } writer.finish(); }
/** * Save this mutable into <code>out</code>. Instantiating a reader from * the generated data will return a reader with the same number of bits * per value. */ public void save(DataOutput out) throws IOException { Writer writer = getWriterNoHeader(out, getFormat(), size(), getBitsPerValue(), DEFAULT_BUFFER_SIZE); writer.writeHeader(); for (int i = 0; i < size(); ++i) { writer.add(get(i)); } writer.finish(); }
/** * Save this mutable into <code>out</code>. Instantiating a reader from * the generated data will return a reader with the same number of bits * per value. */ public void save(DataOutput out) throws IOException { Writer writer = getWriterNoHeader(out, getFormat(), size(), getBitsPerValue(), DEFAULT_BUFFER_SIZE); writer.writeHeader(); for (int i = 0; i < size(); ++i) { writer.add(get(i)); } writer.finish(); }
private void flushFields(int totalFields, int[] fieldNums) throws IOException { final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, PackedInts.bitsRequired(fieldNums.length - 1), 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum); assert fieldNumIndex >= 0; writer.add(fieldNumIndex); } } writer.finish(); }
private void flushFields(int totalFields, int[] fieldNums) throws IOException { final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, PackedInts.bitsRequired(fieldNums.length - 1), 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum); assert fieldNumIndex >= 0; writer.add(fieldNumIndex); } } writer.finish(); }
private void flushFields(int totalFields, int[] fieldNums) throws IOException { final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, totalFields, PackedInts.bitsRequired(fieldNums.length - 1), 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum); assert fieldNumIndex >= 0; writer.add(fieldNumIndex); } } writer.finish(); }
private static void saveInts(int[] values, int length, DataOutput out) throws IOException { assert length > 0; if (length == 1) { out.writeVInt(values[0]); } else { boolean allEqual = true; for (int i = 1; i < length; ++i) { if (values[i] != values[0]) { allEqual = false; break; } } if (allEqual) { out.writeVInt(0); out.writeVInt(values[0]); } else { long max = 0; for (int i = 0; i < length; ++i) { max |= values[i]; } final int bitsRequired = PackedInts.bitsRequired(max); out.writeVInt(bitsRequired); final PackedInts.Writer w = PackedInts.getWriterNoHeader(out, PackedInts.Format.PACKED, length, bitsRequired, 1); for (int i = 0; i < length; ++i) { w.add(values[i]); } w.finish(); } } }
private void flushNumTerms(int totalFields) throws IOException { int maxNumTerms = 0; for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { maxNumTerms |= fd.numTerms; } } final int bitsRequired = PackedInts.bitsRequired(maxNumTerms); vectorsStream.writeVInt(bitsRequired); final PackedInts.Writer writer = PackedInts.getWriterNoHeader( vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { writer.add(fd.numTerms); } } assert writer.ord() == totalFields - 1; writer.finish(); }
/** * Save this mutable into <code>out</code>. Instantiating a reader from * the generated data will return a reader with the same number of bits * per value. */ public void save(DataOutput out) throws IOException { Writer writer = getWriterNoHeader(out, getFormat(), size(), getBitsPerValue(), DEFAULT_BUFFER_SIZE); writer.writeHeader(); for (int i = 0; i < size(); ++i) { writer.add(get(i)); } writer.finish(); }