ByteArrayDataOutput bad = new ByteArrayDataOutput(header); bad.writeByte(ARCS_AS_FIXED_ARRAY); bad.writeVInt(nodeIn.numArcs); bad.writeVInt(maxBytesPerArc); int headerLen = bad.getPosition();
public ByteArrayDataOutput(byte[] bytes, int offset, int len) { reset(bytes, offset, len); }
byte buffer[] = new byte[8]; try { ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); output.reset(buffer); output.writeShort(analyzedLength); output.writeBytes(scratch.bytes(), 0, scratch.length()); output.writeInt(encodeWeight(iterator.weight())); output.writeShort((short) surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeBytes(payload.bytes, payload.offset, payload.length); } else { output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); assert output.getPosition() == requiredLength: output.getPosition() + " vs " + requiredLength; writer.write(buffer, 0, output.getPosition());
ByteArrayDataOutput scratchOutput = new ByteArrayDataOutput(); scratchOutput.reset(scratch.bytes()); scratchOutput.writeVInt(output.ords.get(i)); count++; final int pos = scratchOutput.getPosition(); scratchOutput.writeVInt(count << 1 | (output.includeOrig ? 0 : 1)); final int pos2 = scratchOutput.getPosition(); final int vIntLen = pos2-pos; scratch.setLength(scratchOutput.getPosition());
public void add(int x, int y, int z, int docID) throws IOException { if (pointCount >= maxPointsSortInHeap) { if (writer == null) { switchToOffline(); } scratchBytesOutput.reset(scratchBytes); scratchBytesOutput.writeInt(x); scratchBytesOutput.writeInt(y); scratchBytesOutput.writeInt(z); scratchBytesOutput.writeVInt(docID); scratchBytesOutput.writeVLong(pointCount); writer.write(scratchBytes, 0, scratchBytes.length); } else { // Not too many points added yet, continue using heap: heapWriter.append(x, y, z, pointCount, docID); } pointCount++; }
ByteArrayDataOutput affixWriter = new ByteArrayDataOutput(affixData, currentAffix << 3, numLines << 3); assert affixWriter.getPosition() == currentAffix << 3; String line = reader.readLine(); String ruleArgs[] = line.split("\\s+"); affixWriter.writeShort((short)flag); affixWriter.writeShort((short)stripOrd.intValue()); affixWriter.writeShort((short)patternOrd); affixWriter.writeShort((short)appendFlagsOrd);
protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException { if (spare.length + 8 >= buffer.length) { buffer = ArrayUtil.grow(buffer, spare.length + 8); } output.reset(buffer); output.writeBytes(spare.bytes, spare.offset, spare.length); output.writeLong(weight); writer.write(buffer, 0, output.getPosition()); }
ByteArrayDataOutput out = new ByteArrayDataOutput(byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining()); writeVInts(out, tree.getNextPointerArray(), nodeCount); .get(binding.getPrefix()); int b = namespaces.get(binding.getURI()); out.writeVInt((a << 16) | b); Integer index = attValues.get(tree.getAttributeValueArray()[i]); out.writeVInt(index == null ? 0 : index.intValue()); byteBuffer.position(out.getPosition() - byteBuffer.arrayOffset());
/** If the current segment has too many points then we switchover to temp files / offline sort. */ private void switchToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: tempInput = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "in", ""); writer = new OfflineSorter.ByteSequencesWriter(tempInput); for(int i=0;i<valueCount;i++) { scratchBytesOutput.reset(scratchBytes); scratchBytesOutput.writeLong(heapWriter.values[i]); scratchBytesOutput.writeVInt(heapWriter.docIDs[i]); scratchBytesOutput.writeVLong(i); // TODO: can/should OfflineSorter optimize the fixed-width case? writer.write(scratchBytes, 0, scratchBytes.length); } heapWriter = null; }
private void writeShortDeltas(ByteArrayDataOutput out, short[] shorts, int count) { int k = 0; try { if (formatVersion < 1) { for (int i = 0; i < count; i++) { k = shorts[i] - k; out.writeByte((byte) (k & 0xff)); k = shorts[i]; } } else { for (int i = 0; i < count; i++) { k = shorts[i] - k; out.writeVInt(k); k = shorts[i]; } } } catch (IOException e) { } }
@Override public BytesRef binaryValue() { CollectionUtils.sortAndDedup(values); // here is the trick: // - the first value is zig-zag encoded so that eg. -5 would become positive and would be better compressed by vLong // - for other values, we only encode deltas using vLong final byte[] bytes = new byte[values.size() * ByteUtils.MAX_BYTES_VLONG]; final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); ByteUtils.writeVLong(out, ByteUtils.zigZagEncode(values.get(0))); for (int i = 1; i < values.size(); ++i) { final long delta = values.get(i) - values.get(i - 1); ByteUtils.writeVLong(out, delta); } return new BytesRef(bytes, 0, out.getPosition()); }
private void write(FileChannel channel) throws IOException { byte[] buffer = new byte[BUFFER_SIZE]; final ByteArrayDataOutput out = new ByteArrayDataOutput(buffer); write(out); Channels.writeToChannel(buffer, channel); }
private void writeVInts(ByteArrayDataOutput out, int[] ints, int count) { try { for (int i = 0; i < count; i++) { out.writeVInt(ints[i]); } } catch (IOException e) { } }
private void encodeValues(int count) throws IOException { out.reset(buffer); long lastOrd = 0; for (int i = 0; i < count; i++) { long ord = ords.next().longValue(); out.writeVLong(ord - lastOrd); lastOrd = ord; } }
@Override public BytesRef next() { if (!hasNext()) { throw new NoSuchElementException(); } int count = counts.next().intValue(); int maxSize = count*9; // worst case if (maxSize > buffer.length) { buffer = ArrayUtil.grow(buffer, maxSize); } try { encodeValues(count); } catch (IOException bogus) { throw new RuntimeException(bogus); } ref.bytes = buffer; ref.offset = 0; ref.length = out.getPosition(); return ref; }
@Override protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException { if (spare.length + 5 >= buffer.length) { buffer = ArrayUtil.grow(buffer, spare.length + 5); } output.reset(buffer); output.writeBytes(spare.bytes, spare.offset, spare.length); output.writeByte((byte)0); // separator: not used, just for sort order output.writeInt(encodeWeight(weight)); writer.write(buffer, 0, output.getPosition()); }
ByteArrayDataOutput scratchOutput = new ByteArrayDataOutput(); scratchOutput.reset(scratch.bytes, scratch.offset, scratch.bytes.length); assert scratch.offset == 0; scratchOutput.writeVInt(output.ords.get(i)); count++; final int pos = scratchOutput.getPosition(); scratchOutput.writeVInt(count << 1 | (output.includeOrig ? 0 : 1)); final int pos2 = scratchOutput.getPosition(); final int vIntLen = pos2-pos; scratch.length = scratchOutput.getPosition() - scratch.offset;
/** If the current segment has too many points then we switchover to temp files / offline sort. */ private void switchToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: tempInput = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "in", ""); writer = new OfflineSorter.ByteSequencesWriter(tempInput); for(int i=0;i<pointCount;i++) { scratchBytesOutput.reset(scratchBytes); scratchBytesOutput.writeInt(heapWriter.xs[i]); scratchBytesOutput.writeInt(heapWriter.ys[i]); scratchBytesOutput.writeInt(heapWriter.zs[i]); scratchBytesOutput.writeVInt(heapWriter.docIDs[i]); scratchBytesOutput.writeVLong(i); // TODO: can/should OfflineSorter optimize the fixed-width case? writer.write(scratchBytes, 0, scratchBytes.length); } heapWriter = null; }