@Override public long length() { return compressionDescriptor.getNumberOfElements(); }
public CompressedDataBuffer(Pointer pointer, @NonNull CompressionDescriptor descriptor) { this.compressionDescriptor = descriptor; this.pointer = pointer; this.length = descriptor.getNumberOfElements(); this.elementSize = (byte) descriptor.getOriginalElementSize(); initTypeAndSize(); }
@Override public DataBuffer decompress(DataBuffer buffer) { try { CompressedDataBuffer compressed = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); BytePointer pointer = (BytePointer) compressed.addressPointer(); ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes()); GZIPInputStream gzip = new GZIPInputStream(bis); DataInputStream dis = new DataInputStream(gzip); DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements()); bufferRestored.read(dis); return bufferRestored; } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void write(DataOutputStream out) throws IOException { // logger.info("Writing out CompressedDataBuffer"); // here we should mimic to usual DataBuffer array out.writeUTF(allocationMode.name()); out.writeInt((int) compressionDescriptor.getCompressedLength()); out.writeUTF(Type.COMPRESSED.name()); // at this moment we don't care about mimics anymore //ByteRawIndexer indexer = new ByteRawIndexer((BytePointer) pointer); out.writeUTF(compressionDescriptor.getCompressionAlgorithm()); out.writeLong(compressionDescriptor.getCompressedLength()); out.writeLong(compressionDescriptor.getOriginalLength()); out.writeLong(compressionDescriptor.getNumberOfElements()); // out.write(((BytePointer) pointer).getStringBytes()); for (int x = 0; x < pointer.capacity() * pointer.sizeof(); x++) { byte b = pointer.asByteBuffer().get(x); out.writeByte(b); } }
@Override public long length() { return compressionDescriptor.getNumberOfElements(); }
public CompressedDataBuffer(Pointer pointer, @NonNull CompressionDescriptor descriptor) { this.compressionDescriptor = descriptor; this.pointer = pointer; this.length = descriptor.getNumberOfElements(); this.elementSize = (byte) descriptor.getOriginalElementSize(); initTypeAndSize(); }
@Override public DataBuffer decompress(DataBuffer buffer) { try { CompressedDataBuffer compressed = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); BytePointer pointer = (BytePointer) compressed.addressPointer(); ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes()); GZIPInputStream gzip = new GZIPInputStream(bis); DataInputStream dis = new DataInputStream(gzip); DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements()); bufferRestored.read(dis); return bufferRestored; } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void write(DataOutputStream out) throws IOException { // logger.info("Writing out CompressedDataBuffer"); // here we should mimic to usual DataBuffer array out.writeUTF(allocationMode.name()); out.writeInt((int) compressionDescriptor.getCompressedLength()); out.writeUTF(Type.COMPRESSED.name()); // at this moment we don't care about mimics anymore //ByteRawIndexer indexer = new ByteRawIndexer((BytePointer) pointer); out.writeUTF(compressionDescriptor.getCompressionAlgorithm()); out.writeLong(compressionDescriptor.getCompressedLength()); out.writeLong(compressionDescriptor.getOriginalLength()); out.writeLong(compressionDescriptor.getNumberOfElements()); // out.write(((BytePointer) pointer).getStringBytes()); for (int x = 0; x < pointer.capacity() * pointer.sizeof(); x++) { byte b = pointer.asByteBuffer().get(x); out.writeByte(b); } }
CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); buffer = Nd4j.createBuffer(descriptor.getNumberOfElements(), false);
CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); buffer = Nd4j.createBuffer(descriptor.getNumberOfElements(), false);