/** Reads a string from the buffer as a 1_4 encoded length in bytes followed by the utf8 bytes */ public String getUtf8String() { int stringLength = getInt1_4Bytes(); byte[] stringBytes = new byte[stringLength]; get(stringBytes); return Utf8.toString(stringBytes); }
/** * Reads a 31-bit positive integer from the buffer, which was written using 1 or 4 bytes. * * @return the integer read */ public int getInt1_4Bytes() { byte flagByte = get(); position(position() - 1); if ((flagByte & 0x80) != 0) { //length 4 bytes return getInt() & 0x7FFFFFFF; } else { //length 1 byte return get(); } }
/** * Reads a 30-bit positive integer from the buffer, which was written using 1, 2, or 4 bytes. * * @return the integer read */ public int getInt1_2_4Bytes() { byte flagByte = get(); position(position() - 1); if ((flagByte & 0x80) != 0) { if ((flagByte & 0x40) != 0) { //length 4 bytes return getInt() & 0x3FFFFFFF; } else { //length 2 bytes return getShort() & 0x3FFF; } } else { //length 1 byte return get(); } }
public String getString(FieldBase field) { int length = getInt(null); byte[] stringArray = new byte[length-1]; buf.get(stringArray); getByte(null); return Utf8.toString(stringArray); } public int getInt1_4Bytes(FieldBase field) { return buf.getInt1_4Bytes(); }
public void read(AlternateSpanList altSpanList) { byte type = buf.get(); if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) { throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type); } int numSubTrees = buf.getInt1_2_4Bytes(); for (int i = 0; i < numSubTrees; i++) { double prob = buf.getDouble(); List<SpanNode> list = readSpanList(altSpanList); if (i == 0) { for (SpanNode node : list) { altSpanList.add(node); } altSpanList.setProbability(0, prob); } else { altSpanList.addChildren(i, list, prob); } } }
/** * Reads a 62-bit positive integer from the buffer, which was written using 2, 4, or 8 bytes. * * @return the integer read */ public long getInt2_4_8Bytes() { byte flagByte = get(); position(position() - 1); if ((flagByte & 0x80) != 0) { if ((flagByte & 0x40) != 0) { //length 8 bytes return getLong() & 0x3FFFFFFFFFFFFFFFL; } else { //length 4 bytes return getInt() & 0x3FFFFFFF; } } else { //length 2 bytes return getShort(); } }
public void read(Span span) { byte type = buf.get(); if ((type & Span.ID) != Span.ID) { throw new DeserializationException("Cannot deserialize Span with type " + type); } span.setFrom(buf.getInt1_2_4Bytes()); span.setLength(buf.getInt1_2_4Bytes()); }
public void read(SpanList spanList) { byte type = buf.get(); if ((type & SpanList.ID) != SpanList.ID) { throw new DeserializationException("Cannot deserialize SpanList with type " + type); } List<SpanNode> nodes = readSpanList(spanList); for (SpanNode node : nodes) { spanList.add(node); } }
public void read(AlternateSpanList altSpanList) { byte type = buf.get(); if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) { throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type); } int numSubTrees = buf.getInt1_2_4Bytes(); for (int i = 0; i < numSubTrees; i++) { double prob = buf.getDouble(); List<SpanNode> list = readSpanList(altSpanList); if (i == 0) { for (SpanNode node : list) { altSpanList.add(node); } altSpanList.setProbability(0, prob); } else { altSpanList.addChildren(i, list, prob); } } }
public void read(Span span) { byte type = buf.get(); if ((type & Span.ID) != Span.ID) { throw new DeserializationException("Cannot deserialize Span with type " + type); } span.setFrom(buf.getInt1_2_4Bytes()); span.setLength(buf.getInt1_2_4Bytes()); }
public void read(SpanList spanList) { byte type = buf.get(); if ((type & SpanList.ID) != SpanList.ID) { throw new DeserializationException("Cannot deserialize SpanList with type " + type); } List<SpanNode> nodes = readSpanList(spanList); for (SpanNode node : nodes) { spanList.add(node); } }
@Override public void read(FieldBase field, ReferenceFieldValue value) { final boolean documentIdPresent = (buf.get() != 0); if (documentIdPresent) { value.assign(readDocumentId()); } else { value.clear(); } }
@Override public void read(FieldBase field, ReferenceFieldValue value) { final boolean documentIdPresent = (buf.get() != 0); if (documentIdPresent) { value.assign(readDocumentId()); } else { value.clear(); } }
private SpanNode readSpanNode() { byte type = buf.get(); buf.position(buf.position() - 1); SpanNode retval; if ((type & Span.ID) == Span.ID) { retval = new Span(); if (spanNodes != null) { spanNodes.add(retval); } read((Span) retval); } else if ((type & SpanList.ID) == SpanList.ID) { retval = new SpanList(); if (spanNodes != null) { spanNodes.add(retval); } read((SpanList) retval); } else if ((type & AlternateSpanList.ID) == AlternateSpanList.ID) { retval = new AlternateSpanList(); if (spanNodes != null) { spanNodes.add(retval); } read((AlternateSpanList) retval); } else { throw new DeserializationException("Cannot read SpanNode of type " + type); } return retval; }
private SpanNode readSpanNode() { byte type = buf.get(); buf.position(buf.position() - 1); SpanNode retval; if ((type & Span.ID) == Span.ID) { retval = new Span(); if (spanNodes != null) { spanNodes.add(retval); } read((Span) retval); } else if ((type & SpanList.ID) == SpanList.ID) { retval = new SpanList(); if (spanNodes != null) { spanNodes.add(retval); } read((SpanList) retval); } else if ((type & AlternateSpanList.ID) == AlternateSpanList.ID) { retval = new AlternateSpanList(); if (spanNodes != null) { spanNodes.add(retval); } read((AlternateSpanList) retval); } else { throw new DeserializationException("Cannot read SpanNode of type " + type); } return retval; }
public byte[] serialize() { BufferSerializer out = new BufferSerializer(new GrowableByteBuffer()); out.putInt(null, distributionBits); out.putLong(null, bucketCursor); out.putLong(null, finishedBucketCount); out.putLong(null, totalBucketCount); out.putInt(null, buckets.size()); // Append individual bucket progress for (Map.Entry<BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) { out.putLong(null, keyToBucketId(entry.getKey().getKey())); out.putLong(null, entry.getValue().getProgress().getRawId()); } byte[] ret = new byte[out.getBuf().position()]; out.getBuf().rewind(); out.getBuf().get(ret); return ret; }
public void serialize(OutputStream out) throws SerializationException { DocumentSerializer writer = DocumentSerializerFactory.create6(new GrowableByteBuffer(64 * 1024, 2.0f)); writer.write(this); GrowableByteBuffer data = writer.getBuf(); byte[] array; if (data.hasArray()) { //just get the array array = data.array(); } else { //copy the bytebuffer into the array array = new byte[data.position()]; int endPos = data.position(); data.position(0); data.get(array); data.position(endPos); } try { out.write(array, 0, data.position()); } catch (IOException ioe) { throw new SerializationException(ioe); } }
public static byte[] encode(Tensor tensor) { GrowableByteBuffer buffer = new GrowableByteBuffer(); if (tensor instanceof MixedTensor) { buffer.putInt1_4Bytes(MIXED_BINARY_FORMAT_TYPE); new MixedBinaryFormat().encode(buffer, tensor); } else if (tensor instanceof IndexedTensor) { buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE); new DenseBinaryFormat().encode(buffer, tensor); } else { buffer.putInt1_4Bytes(SPARSE_BINARY_FORMAT_TYPE); new SparseBinaryFormat().encode(buffer, tensor); } buffer.flip(); byte[] result = new byte[buffer.remaining()]; buffer.get(result); return result; }