public void serialize(OutputStream out) throws SerializationException { DocumentSerializer writer = DocumentSerializerFactory.create6(new GrowableByteBuffer(64 * 1024, 2.0f)); writer.write(this); GrowableByteBuffer data = writer.getBuf(); byte[] array; if (data.hasArray()) { //just get the array array = data.array(); } else { //copy the bytebuffer into the array array = new byte[data.position()]; int endPos = data.position(); data.position(0); data.get(array); data.position(endPos); } try { out.write(array, 0, data.position()); } catch (IOException ioe) { throw new SerializationException(ioe); } }
byte[] serializeDocument(Document doc) { if (doc != null) { GrowableByteBuffer buf = new GrowableByteBuffer(); DocumentSerializer serializer = DocumentSerializerFactory.create42(buf); doc.serialize(serializer); buf.flip(); return buf.array(); } else { return new byte[0]; } }
public int getInt(FieldBase field) { return buf.getByteBuffer().getInt(); } public long getLong(FieldBase field) { return buf.getByteBuffer().getLong(); }
public void serialize(GrowableByteBuffer buf) { ByteOrder originalOrder = buf.order(); buf.order(ByteOrder.LITTLE_ENDIAN); buf.putLong(timestamp); // 8 buf.putInt(headerPos); // 12 buf.putInt(headerLen); // 16 buf.putInt(bodyPos); // 20 buf.putInt(bodyLen); // 24 buf.putInt(flags); // 28 (written as little-endian int, this is on purpose) buf.putInt(0); // 32 buf.order(originalOrder); } }
/** * Reads a 31-bit positive integer from the buffer, which was written using 1 or 4 bytes. * * @return the integer read */ public int getInt1_4Bytes() { byte flagByte = get(); position(position() - 1); if ((flagByte & 0x80) != 0) { //length 4 bytes return getInt() & 0x7FFFFFFF; } else { //length 1 byte return get(); } }
GrowableByteBuffer buffer = new GrowableByteBuffer(4096, 2.0f); buf = buffer; int startPos = buffer.position(); value.getValue().serialize(value.getKey(), this); fieldLengths.add(buffer.position() - startPos); fieldIds.add(value.getKey().getId(s.getVersion())); buffer.flip(); buf = bigBuffer; int uncompressedSize = buffer.remaining(); Compressor.Compression compression = s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining()); int lenPos = buf.position(); putInt(null, 0); // Move back to this after compression is done. buf.put(compression.type().getCode()); buf.putInt2_4_8Bytes(uncompressedSize); buf.putInt1_4Bytes(s.getFieldCount()); int pos = buf.position(); if (compression.data() != null && compression.type().isCompressed()) { put(null, compression.data()); } else { put(null, buffer.getByteBuffer());
public static byte[] encode(Tensor tensor) { GrowableByteBuffer buffer = new GrowableByteBuffer(); if (tensor instanceof MixedTensor) { buffer.putInt1_4Bytes(MIXED_BINARY_FORMAT_TYPE); new MixedBinaryFormat().encode(buffer, tensor); } else if (tensor instanceof IndexedTensor) { buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE); new DenseBinaryFormat().encode(buffer, tensor); } else { buffer.putInt1_4Bytes(SPARSE_BINARY_FORMAT_TYPE); new SparseBinaryFormat().encode(buffer, tensor); } buffer.flip(); byte[] result = new byte[buffer.remaining()]; buffer.get(result); return result; }
public GrowableByteBuffer duplicate() { return new GrowableByteBuffer(buffer.duplicate(), growFactor); } public boolean equals(Object obj) {
BufferSerializer gbuf = new BufferSerializer(new GrowableByteBuffer()); gbuf.putInt(null, groupingList.size()); for (Grouping g: groupingList){ g.serialize(gbuf); gbuf.getBuf().flip(); byte[] blob = new byte [gbuf.getBuf().limit()]; gbuf.getBuf().get(blob); buffer.putInt(blob.length); buffer.put(blob);
/** * Reads a 30-bit positive integer from the buffer, which was written using 1, 2, or 4 bytes. * * @return the integer read */ public int getInt1_2_4Bytes() { byte flagByte = get(); position(position() - 1); if ((flagByte & 0x80) != 0) { if ((flagByte & 0x40) != 0) { //length 4 bytes return getInt() & 0x3FFFFFFF; } else { //length 2 bytes return getShort() & 0x3FFF; } } else { //length 1 byte return get(); } }
public byte[] serialize() { BufferSerializer out = new BufferSerializer(new GrowableByteBuffer()); out.putInt(null, distributionBits); out.putLong(null, bucketCursor); out.putLong(null, finishedBucketCount); out.putLong(null, totalBucketCount); out.putInt(null, buckets.size()); // Append individual bucket progress for (Map.Entry<BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) { out.putLong(null, keyToBucketId(entry.getKey().getKey())); out.putLong(null, entry.getValue().getProgress().getRawId()); } byte[] ret = new byte[out.getBuf().position()]; out.getBuf().rewind(); out.getBuf().get(ret); return ret; }
public void write(Annotation annotation) { buf.putInt(annotation.getType().getId()); //name hash features |= ((byte) 2); buf.put(features); int posBeforeSize = buf.position(); buf.putInt1_2_4BytesAs4(0); int spanNodeId = annotation.getSpanNode().getScratchId(); if (spanNodeId >= 0) { buf.putInt1_2_4Bytes(spanNodeId); } else { throw new SerializationException("Could not serialize annotation, associated SpanNode not found (" + annotation + ")"); buf.putInt(annotation.getType().getDataType().getId()); annotation.getFieldValue().serialize(this); int end = buf.position(); buf.position(posBeforeSize); buf.putInt1_2_4BytesAs4(end - posBeforeSize - 4); buf.position(end);
public void write(FieldBase field, Document doc) { buf.putShort(Document.SERIALIZED_VERSION); int lenPos = buf.position(); buf.putInt(0); contents |= 0x4; // Indicate we have a body buf.put(contents); body.serialize(null, this); int finalPos = buf.position(); buf.position(lenPos); buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version buf.position(finalPos);
/** * Returns the serialized size of the given {@link Document}. Please note that this method performs actual * serialization of the document, but simply return the size of the final {@link GrowableByteBuffer}. If you need * the buffer itself, do NOT use this method. * * @param doc The Document whose size to calculate. * @return The size in bytes. */ public static long getSerializedSize(Document doc) { DocumentSerializer serializer = new VespaDocumentSerializer6(new GrowableByteBuffer()); serializer.write(doc); return serializer.getBuf().position(); }
public void read(AlternateSpanList altSpanList) { byte type = buf.get(); if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) { throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type); } int numSubTrees = buf.getInt1_2_4Bytes(); for (int i = 0; i < numSubTrees; i++) { double prob = buf.getDouble(); List<SpanNode> list = readSpanList(altSpanList); if (i == 0) { for (SpanNode node : list) { altSpanList.add(node); } altSpanList.setProbability(0, prob); } else { altSpanList.addChildren(i, list, prob); } } }
/** * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes. * * @param number the integer to write */ public void putInt1_4Bytes(int number) { if (number < 0) { throw new IllegalArgumentException("Cannot encode negative number"); } //no need to check upper boundary, since INT_MAX == 2^31 if (number < 0x80) { //length 1 byte put((byte) number); } else { //length 4 bytes putInt(number | 0x80000000); } }
/** * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. * * @param number the integer to write */ public void putInt2_4_8Bytes(long number) { if (number < 0L) { throw new IllegalArgumentException("Cannot encode negative number."); } else if (number > 0x3FFFFFFFFFFFFFFFL) { throw new IllegalArgumentException("Cannot encode number larger than 2^62."); } if (number < 0x8000L) { //length 2 bytes putShort((short) number); } else if (number < 0x40000000L) { //length 4 bytes putInt(((int) number) | 0x80000000); } else { //length 8 bytes putLong(number | 0xC000000000000000L); } }