public int getInt1_4Bytes(FieldBase field) { return buf.getInt1_4Bytes(); } public int getInt1_2_4Bytes(FieldBase field) { return buf.getInt1_2_4Bytes(); }
private TensorType decodeType(GrowableByteBuffer buffer) { int dimensionCount = buffer.getInt1_4Bytes(); TensorType.Builder builder = new TensorType.Builder(); for (int i = 0; i < dimensionCount; i++) builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); // XXX: Size truncation return builder.build(); }
/** Reads a string from the buffer as a 1_4 encoded length in bytes followed by the utf8 bytes */ public String getUtf8String() { int stringLength = getInt1_4Bytes(); byte[] stringBytes = new byte[stringLength]; get(stringBytes); return Utf8.toString(stringBytes); }
private TensorType decodeType(GrowableByteBuffer buffer) { TensorType.Builder builder = new TensorType.Builder(); int numMappedDimensions = buffer.getInt1_4Bytes(); for (int i = 0; i < numMappedDimensions; ++i) { builder.mapped(buffer.getUtf8String()); } int numIndexedDimensions = buffer.getInt1_4Bytes(); for (int i = 0; i < numIndexedDimensions; ++i) { builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); // XXX: Size truncation } return builder.build(); }
private void decodeCells(GrowableByteBuffer buffer, Tensor.Builder builder, TensorType type) { long numCells = buffer.getInt1_4Bytes(); // XXX: Size truncation for (long i = 0; i < numCells; ++i) { Tensor.Builder.CellBuilder cellBuilder = builder.cell(); decodeAddress(buffer, cellBuilder, type); cellBuilder.value(buffer.getDouble()); } }
private TensorType decodeType(GrowableByteBuffer buffer) { int numDimensions = buffer.getInt1_4Bytes(); TensorType.Builder builder = new TensorType.Builder(); for (int i = 0; i < numDimensions; ++i) builder.mapped(buffer.getUtf8String()); return builder.build(); }
@Override public void read(FieldBase field, TensorFieldValue value) { int encodedTensorLength = buf.getInt1_4Bytes(); if (encodedTensorLength > 0) { byte[] encodedTensor = getBytes(null, encodedTensorLength); value.assign(TypedBinaryFormat.decode(Optional.of(value.getDataType().getTensorType()), GrowableByteBuffer.wrap(encodedTensor))); } else { value.clear(); } }
@Override public void read(FieldBase field, TensorFieldValue value) { int encodedTensorLength = buf.getInt1_4Bytes(); if (encodedTensorLength > 0) { byte[] encodedTensor = getBytes(null, encodedTensorLength); value.assign(TypedBinaryFormat.decode(Optional.of(value.getDataType().getTensorType()), GrowableByteBuffer.wrap(encodedTensor))); } else { value.clear(); } }
/** * Decode some data to a tensor * * @param type the type to decode and validate to, or empty to use the type given in the data * @param buffer the buffer containing the data, use GrowableByteByffer.wrap(byte[]) if you have a byte array * @return the resulting tensor * @throws IllegalArgumentException if the tensor data was invalid */ public static Tensor decode(Optional<TensorType> type, GrowableByteBuffer buffer) { int formatType = buffer.getInt1_4Bytes(); switch (formatType) { case MIXED_BINARY_FORMAT_TYPE: return new MixedBinaryFormat().decode(type, buffer); case SPARSE_BINARY_FORMAT_TYPE: return new SparseBinaryFormat().decode(type, buffer); case DENSE_BINARY_FORMAT_TYPE: return new DenseBinaryFormat().decode(type, buffer); default: throw new IllegalArgumentException("Binary format type " + formatType + " is unknown"); } }
private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type) { List<TensorType.Dimension> sparseDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); TensorType sparseType = MixedTensor.createPartialType(sparseDimensions); long denseSubspaceSize = builder.denseSubspaceSize(); int numBlocks = 1; if (sparseDimensions.size() > 0) { numBlocks = buffer.getInt1_4Bytes(); } double[] denseSubspace = new double[(int)denseSubspaceSize]; for (int i = 0; i < numBlocks; ++i) { TensorAddress.Builder sparseAddress = new TensorAddress.Builder(sparseType); for (TensorType.Dimension sparseDimension : sparseDimensions) { sparseAddress.add(sparseDimension.name(), buffer.getUtf8String()); } for (long denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) { denseSubspace[(int)denseOffset] = buffer.getDouble(); } builder.block(sparseAddress.build(), denseSubspace); } }