BytesStore emptyBytes = new BytesStore(10); int numBytes = in.readVInt(); emptyBytes.copyBytes(in, numBytes); BytesReader reader = emptyBytes.getReverseReader(); if (numBytes > 1 << maxBlockBits) { bytes = new BytesStore(in, numBytes, 1<<maxBlockBits); bytesArray = null; } else {
void finish(long newStartNode) throws IOException { assert newStartNode <= bytes.getPosition(); if (startNode != -1) { throw new IllegalStateException("already finished"); } if (newStartNode == FINAL_END_NODE && emptyOutput != null) { newStartNode = 0; } startNode = newStartNode; bytes.finish(); cacheRootArcs(); }
@Override public long ramBytesUsed() { long size = BASE_RAM_BYTES_USED; if (bytesArray != null) { size += bytesArray.length; } else { size += bytes.ramBytesUsed(); } size += cachedArcsBytesUsed; return size; }
FST(INPUT_TYPE inputType, Outputs<T> outputs, int bytesPageBits) { this.inputType = inputType; this.outputs = outputs; version = VERSION_CURRENT; bytesArray = null; bytes = new BytesStore(bytesPageBits); // pad: ensure no node gets address 0 which is reserved to mean // the stop state w/ no arcs bytes.writeByte((byte) 0); emptyOutput = null; }
final long startAddress = builder.bytes.getPosition(); long lastArcStart = builder.bytes.getPosition(); int maxBytesPerArc = 0; for(int arcIdx=0;arcIdx<nodeIn.numArcs;arcIdx++) { builder.bytes.writeByte((byte) flags); writeLabel(builder.bytes, arc.label); builder.bytes.writeVLong(target.node); builder.reusedBytesPerArc[arcIdx] = (int) (builder.bytes.getPosition() - lastArcStart); lastArcStart = builder.bytes.getPosition(); maxBytesPerArc = Math.max(maxBytesPerArc, builder.reusedBytesPerArc[arcIdx]); long srcPos = builder.bytes.getPosition(); long destPos = fixedArrayStart + nodeIn.numArcs*maxBytesPerArc; assert destPos >= srcPos; if (destPos > srcPos) { builder.bytes.skipBytes((int) (destPos - srcPos)); for(int arcIdx=nodeIn.numArcs-1;arcIdx>=0;arcIdx--) { destPos -= maxBytesPerArc; builder.bytes.copyBytes(srcPos, destPos, builder.reusedBytesPerArc[arcIdx]); builder.bytes.writeBytes(startAddress, header, 0, headerLen); final long thisNodeAddress = builder.bytes.getPosition()-1;
PackedInts.bitsRequired(builder.bytes.getPosition()), (int) (1 + builder.nodeCount), acceptableOverheadRatio); newNodeAddress.set(node, 1 + builder.bytes.getPosition() - nodeAddress.get(node)); fst = new FST<>(inputType, outputs, builder.bytes.getBlockBits()); writer.writeByte((byte) 0); final long address = writer.getPosition(); writer.writeByte(ARCS_AS_FIXED_ARRAY); writer.writeVInt(arc.numArcs); writer.writeVInt(bytesPerArc); final long arcStartPos = writer.getPosition(); nodeArcCount++; long delta = newNodeAddress.get((int) arc.target) + addressError - writer.getPosition() - 2; if (delta < 0) { writer.writeByte(flags); long delta = newNodeAddress.get((int) arc.target) + addressError - writer.getPosition(); if (delta < 0) { anyNegDelta = true; writer.writeVLong(delta); if (!retry) { deltaCount++;
BytesStore emptyBytes = new BytesStore(10); int numBytes = in.readVInt(); emptyBytes.copyBytes(in, numBytes); reader = emptyBytes.getForwardReader(); } else { reader = emptyBytes.getReverseReader(); if (numBytes > 1 << maxBlockBits) { bytes = new BytesStore(in, numBytes, 1<<maxBlockBits); bytesArray = null; } else {
/** Pos must be less than the max position written so far! * Ie, you cannot "grow" the file with this! */ public void truncate(long newLen) { assert newLen <= getPosition(); assert newLen >= 0; int blockIndex = (int) (newLen >> blockBits); nextWrite = (int) (newLen & blockMask); if (nextWrite == 0) { blockIndex--; nextWrite = blockSize; } blocks.subList(blockIndex+1, blocks.size()).clear(); if (newLen == 0) { current = null; } else { current = blocks.get(blockIndex); } assert newLen == getPosition(); }
out.writeVLong(startNode); if (bytes != null) { long numBytes = bytes.getPosition(); out.writeVLong(numBytes); bytes.writeTo(out); } else { assert bytesArray != null;
public FST.BytesReader getReverseReader() { return getReverseReader(true); }
/** Returns a {@link BytesReader} for this FST, positioned at * position 0. */ public BytesReader getBytesReader() { if (packed) { if (bytesArray != null) { return new ForwardBytesReader(bytesArray); } else { return bytes.getForwardReader(); } } else { if (bytesArray != null) { return new ReverseBytesReader(bytesArray); } else { return bytes.getReverseReader(); } } }
writeBytes(dest, block, downTo-len, len); break; } else { writeBytes(dest + len, block, 0, downTo); blockIndex--; block = blocks.get(blockIndex);
private FST(INPUT_TYPE inputType, Outputs<T> outputs, int bytesPageBits) { version = VERSION_CURRENT; packed = true; this.inputType = inputType; bytesArray = null; bytes = new BytesStore(bytesPageBits); this.outputs = outputs; }
final long startAddress = builder.bytes.getPosition(); long lastArcStart = builder.bytes.getPosition(); int maxBytesPerArc = 0; for(int arcIdx=0;arcIdx<nodeIn.numArcs;arcIdx++) { builder.bytes.writeByte((byte) flags); writeLabel(builder.bytes, arc.label); builder.bytes.writeVLong(target.node); builder.reusedBytesPerArc[arcIdx] = (int) (builder.bytes.getPosition() - lastArcStart); lastArcStart = builder.bytes.getPosition(); maxBytesPerArc = Math.max(maxBytesPerArc, builder.reusedBytesPerArc[arcIdx]); long srcPos = builder.bytes.getPosition(); long destPos = fixedArrayStart + nodeIn.numArcs*maxBytesPerArc; assert destPos >= srcPos; if (destPos > srcPos) { builder.bytes.skipBytes((int) (destPos - srcPos)); for(int arcIdx=nodeIn.numArcs-1;arcIdx>=0;arcIdx--) { destPos -= maxBytesPerArc; builder.bytes.copyBytes(srcPos, destPos, builder.reusedBytesPerArc[arcIdx]); builder.bytes.writeBytes(startAddress, header, 0, headerLen); final long thisNodeAddress = builder.bytes.getPosition()-1;
PackedInts.bitsRequired(builder.bytes.getPosition()), (int) (1 + builder.nodeCount), acceptableOverheadRatio); newNodeAddress.set(node, 1 + builder.bytes.getPosition() - nodeAddress.get(node)); fst = new FST<>(inputType, outputs, builder.bytes.getBlockBits()); writer.writeByte((byte) 0); final long address = writer.getPosition(); writer.writeByte(ARCS_AS_FIXED_ARRAY); writer.writeVInt(arc.numArcs); writer.writeVInt(bytesPerArc); final long arcStartPos = writer.getPosition(); nodeArcCount++; long delta = newNodeAddress.get((int) arc.target) + addressError - writer.getPosition() - 2; if (delta < 0) { writer.writeByte(flags); long delta = newNodeAddress.get((int) arc.target) + addressError - writer.getPosition(); if (delta < 0) { anyNegDelta = true; writer.writeVLong(delta); if (!retry) { deltaCount++;
BytesStore emptyBytes = new BytesStore(10); int numBytes = in.readVInt(); emptyBytes.copyBytes(in, numBytes); reader = emptyBytes.getForwardReader(); } else { reader = emptyBytes.getReverseReader(); if (numBytes > 1 << maxBlockBits) { bytes = new BytesStore(in, numBytes, 1<<maxBlockBits); bytesArray = null; } else {
FST(INPUT_TYPE inputType, Outputs<T> outputs, int bytesPageBits) { this.inputType = inputType; this.outputs = outputs; version = VERSION_CURRENT; bytesArray = null; bytes = new BytesStore(bytesPageBits); // pad: ensure no node gets address 0 which is reserved to mean // the stop state w/ no arcs bytes.writeByte((byte) 0); emptyOutput = null; }
assert destPos < getPosition();
out.writeVLong(startNode); if (bytes != null) { long numBytes = bytes.getPosition(); out.writeVLong(numBytes); bytes.writeTo(out); } else { assert bytesArray != null;
/** Returns a {@link BytesReader} for this FST, positioned at * position 0. */ public BytesReader getBytesReader() { if (bytesArray != null) { return new ReverseBytesReader(bytesArray); } else { return bytes.getReverseReader(); } }