@Override public DataBuffer decompress(DataBuffer buffer) { try { CompressedDataBuffer compressed = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); BytePointer pointer = (BytePointer) compressed.addressPointer(); ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes()); GZIPInputStream gzip = new GZIPInputStream(bis); DataInputStream dis = new DataInputStream(gzip); DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements()); bufferRestored.read(dis); return bufferRestored; } catch (Exception e) { throw new RuntimeException(e); } }
/** * Decompress the given databuffer * @param buffer the databuffer to compress * @return the decompressed databuffer */ public DataBuffer decompress(DataBuffer buffer) { if (buffer.dataType() != DataBuffer.Type.COMPRESSED) throw new IllegalStateException("You can't decompress DataBuffer with dataType of: " + buffer.dataType()); CompressedDataBuffer comp = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(buffer); }
/** * * @param array * @return */ public INDArray decompress(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return array; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array); }
/** * in place decompression of the given * ndarray. If the ndarray isn't compressed * this will do nothing * @param array the array to decompressed * if it is comprssed */ public void decompressi(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); codecs.get(descriptor.getCompressionAlgorithm()).decompressi(array); }
CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor(); ByteBuffer codecByteBuffer = descriptor.toByteBuffer(); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
} else { CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data(); CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor(); ByteBuffer codecByteBuffer = descriptor.toByteBuffer(); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
@Override public DataBuffer decompress(DataBuffer buffer) { try { CompressedDataBuffer compressed = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); BytePointer pointer = (BytePointer) compressed.addressPointer(); ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes()); GZIPInputStream gzip = new GZIPInputStream(bis); DataInputStream dis = new DataInputStream(gzip); DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements()); bufferRestored.read(dis); return bufferRestored; } catch (Exception e) { throw new RuntimeException(e); } }
/** * Decompress the given databuffer * @param buffer the databuffer to compress * @return the decompressed databuffer */ public DataBuffer decompress(DataBuffer buffer) { if (buffer.dataType() != DataBuffer.Type.COMPRESSED) throw new IllegalStateException("You can't decompress DataBuffer with dataType of: " + buffer.dataType()); CompressedDataBuffer comp = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(buffer); }
/** * * @param array * @return */ public INDArray decompress(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return array; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array); }
/** * in place decompression of the given * ndarray. If the ndarray isn't compressed * this will do nothing * @param array the array to decompressed * if it is comprssed */ public void decompressi(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); codecs.get(descriptor.getCompressionAlgorithm()).decompressi(array); }
CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor(); ByteBuffer codecByteBuffer = descriptor.toByteBuffer(); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
} else { CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data(); CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor(); ByteBuffer codecByteBuffer = descriptor.toByteBuffer(); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
@Override public void convertDataEx(DataBuffer.TypeEx typeSrc, Pointer source, DataBuffer.TypeEx typeDst, DataBuffer buffer) { Pointer srcPtr = null; Pointer dstPtr = null; long size = 0; long ssize = 0; val stream = ((CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext()).getOldStream(); if (buffer instanceof CompressedDataBuffer) { // compressing size = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getCompressedLength(); ssize = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getOriginalLength(); srcPtr = nativeOps.mallocDevice(ssize, null, 0); dstPtr = nativeOps.mallocDevice(size, null, 0); nativeOps.memcpyAsync(srcPtr, source, ssize, CudaConstants.cudaMemcpyHostToDevice, stream); } else { // decompressing throw new UnsupportedOperationException(); } convertDataEx(typeSrc, srcPtr, typeDst, dstPtr, buffer.length()); nativeOps.memcpyAsync(buffer.addressPointer(), dstPtr, size, CudaConstants.cudaMemcpyHostToHost, stream); stream.synchronize(); if (buffer instanceof CompressedDataBuffer) { nativeOps.freeDevice(srcPtr, null); nativeOps.freeDevice(dstPtr, null); } }
case COMPRESSED: CompressedDataBuffer compressedData = (CompressedDataBuffer)data; CompressionDescriptor desc = compressedData.getCompressionDescriptor(); String algo = desc.getCompressionAlgorithm(); switch (algo) {
val size = ((CompressedDataBuffer) source).getCompressionDescriptor().getCompressedLength(); srcPtr = ws.alloc(size, MemoryKind.DEVICE, DataBuffer.Type.HALF, false); nativeOps.memcpyAsync(srcPtr, source.addressPointer(), size, CudaConstants.cudaMemcpyHostToHost, stream); val size = ((CompressedDataBuffer) target).getCompressionDescriptor().getCompressedLength(); dstPtr = ws.alloc(size, MemoryKind.DEVICE, DataBuffer.Type.HALF, false); val size = ((CompressedDataBuffer) source).getCompressionDescriptor().getCompressedLength(); srcPtr = nativeOps.mallocDevice(size, null, 0); nativeOps.memcpyAsync(srcPtr, source.addressPointer(), size, CudaConstants.cudaMemcpyHostToHost, stream); val size = ((CompressedDataBuffer) target).getCompressionDescriptor().getCompressedLength(); dstPtr = nativeOps.mallocDevice(size, null, 0);
} else { CompressedDataBuffer compressed = (CompressedDataBuffer) source; CompressionDescriptor descriptor = compressed.getCompressionDescriptor();
} else { CompressedDataBuffer compressed = (CompressedDataBuffer) source; CompressionDescriptor descriptor = compressed.getCompressionDescriptor();