@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length); return buffer; } }
@Override public void write(DataOutputStream out) throws IOException { // logger.info("Writing out CompressedDataBuffer"); // here we should mimic to usual DataBuffer array out.writeUTF(allocationMode.name()); out.writeInt((int) compressionDescriptor.getCompressedLength()); out.writeUTF(Type.COMPRESSED.name()); // at this moment we don't care about mimics anymore //ByteRawIndexer indexer = new ByteRawIndexer((BytePointer) pointer); out.writeUTF(compressionDescriptor.getCompressionAlgorithm()); out.writeLong(compressionDescriptor.getCompressedLength()); out.writeLong(compressionDescriptor.getOriginalLength()); out.writeLong(compressionDescriptor.getNumberOfElements()); // out.write(((BytePointer) pointer).getStringBytes()); for (int x = 0; x < pointer.capacity() * pointer.sizeof(); x++) { byte b = pointer.asByteBuffer().get(x); out.writeByte(b); } }
public CompressedDataBuffer(Pointer pointer, @NonNull CompressionDescriptor descriptor) { this.compressionDescriptor = descriptor; this.pointer = pointer; this.length = descriptor.getNumberOfElements(); this.elementSize = (byte) descriptor.getOriginalElementSize(); initTypeAndSize(); }
CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(compressedLength); descriptor.setCompressionAlgorithm(compressionAlgorithm); descriptor.setOriginalLength(originalLength); descriptor.setNumberOfElements(numberOfElements); return new CompressedDataBuffer(pointer, descriptor); } catch (Exception e) {
CompressionDescriptor descriptor = new CompressionDescriptor(source, typeDst.name()); descriptor.setCompressionType(CompressionType.LOSSY); descriptor.setCompressedLength(source.length() * elementSize); buffer = new CompressedDataBuffer(pointer, descriptor); } else { CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); buffer = Nd4j.createBuffer(descriptor.getNumberOfElements(), false);
@Override public DataBuffer compress(DataBuffer buffer) { try { ByteArrayOutputStream stream = new ByteArrayOutputStream(); GZIPOutputStream gzip = new GZIPOutputStream(stream); DataOutputStream dos = new DataOutputStream(gzip); buffer.write(dos); dos.flush(); dos.close(); byte[] bytes = stream.toByteArray(); // logger.info("Bytes: {}", Arrays.toString(bytes)); BytePointer pointer = new BytePointer(bytes); CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); descriptor.setCompressedLength(bytes.length); CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor); return result; } catch (Exception e) { throw new RuntimeException(e); } }
@Override public CompressionDescriptor clone() { CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.compressionType = this.compressionType; descriptor.compressionAlgorithm = this.compressionAlgorithm; descriptor.originalLength = this.originalLength; descriptor.compressedLength = this.compressedLength; descriptor.numberOfElements = this.numberOfElements; descriptor.originalElementSize = this.originalElementSize; return descriptor; } }
@Override public long length() { return compressionDescriptor.getNumberOfElements(); }
@Override public void convertDataEx(DataBuffer.TypeEx typeSrc, Pointer source, DataBuffer.TypeEx typeDst, DataBuffer buffer) { Pointer srcPtr = null; Pointer dstPtr = null; long size = 0; long ssize = 0; val stream = ((CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext()).getOldStream(); if (buffer instanceof CompressedDataBuffer) { // compressing size = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getCompressedLength(); ssize = ((CompressedDataBuffer) buffer).getCompressionDescriptor().getOriginalLength(); srcPtr = nativeOps.mallocDevice(ssize, null, 0); dstPtr = nativeOps.mallocDevice(size, null, 0); nativeOps.memcpyAsync(srcPtr, source, ssize, CudaConstants.cudaMemcpyHostToDevice, stream); } else { // decompressing throw new UnsupportedOperationException(); } convertDataEx(typeSrc, srcPtr, typeDst, dstPtr, buffer.length()); nativeOps.memcpyAsync(buffer.addressPointer(), dstPtr, size, CudaConstants.cudaMemcpyHostToHost, stream); stream.synchronize(); if (buffer instanceof CompressedDataBuffer) { nativeOps.freeDevice(srcPtr, null); nativeOps.freeDevice(dstPtr, null); } }
/** * Decompress the given databuffer * @param buffer the databuffer to compress * @return the decompressed databuffer */ public DataBuffer decompress(DataBuffer buffer) { if (buffer.dataType() != DataBuffer.Type.COMPRESSED) throw new IllegalStateException("You can't decompress DataBuffer with dataType of: " + buffer.dataType()); CompressedDataBuffer comp = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(buffer); }
@Override public DataBuffer dup() { Pointer nPtr = new BytePointer(compressionDescriptor.getCompressedLength()); val perfD = PerformanceTracker.getInstance().helperStartTransaction(); Pointer.memcpy(nPtr, pointer, compressionDescriptor.getCompressedLength()); PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, compressionDescriptor.getCompressedLength(), MemcpyDirection.HOST_TO_HOST); CompressionDescriptor nDesc = compressionDescriptor.clone(); CompressedDataBuffer nBuf = new CompressedDataBuffer(nPtr, nDesc); return nBuf; }
return Pair.of(arr, byteBuffer); } else { CompressionDescriptor compressionDescriptor = CompressionDescriptor.fromByteBuffer(byteBuffer); ByteBuffer slice = byteBuffer.slice(); int compressLength = (int) compressionDescriptor.getCompressedLength(); byteBuffer.position(byteBuffer.position() + compressLength); return Pair.of(arr, byteBuffer);
val size = ((CompressedDataBuffer) source).getCompressionDescriptor().getCompressedLength(); srcPtr = ws.alloc(size, MemoryKind.DEVICE, DataBuffer.Type.HALF, false); nativeOps.memcpyAsync(srcPtr, source.addressPointer(), size, CudaConstants.cudaMemcpyHostToHost, stream); val size = ((CompressedDataBuffer) target).getCompressionDescriptor().getCompressedLength(); dstPtr = ws.alloc(size, MemoryKind.DEVICE, DataBuffer.Type.HALF, false); val size = ((CompressedDataBuffer) source).getCompressionDescriptor().getCompressedLength(); srcPtr = nativeOps.mallocDevice(size, null, 0); nativeOps.memcpyAsync(srcPtr, source.addressPointer(), size, CudaConstants.cudaMemcpyHostToHost, stream); val size = ((CompressedDataBuffer) target).getCompressionDescriptor().getCompressedLength(); dstPtr = nativeOps.mallocDevice(size, null, 0);
CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(compressedLength); descriptor.setCompressionAlgorithm(compressionAlgorithm); descriptor.setOriginalLength(originalLength); descriptor.setNumberOfElements(numberOfElements);
CompressionDescriptor descriptor = new CompressionDescriptor(source, typeDst.name()); descriptor.setCompressionType(CompressionType.LOSSY); descriptor.setCompressedLength(source.length() * elementSize); buffer = new CompressedDataBuffer(pointer, descriptor); } else { CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); buffer = Nd4j.createBuffer(descriptor.getNumberOfElements(), false);
@Override public DataBuffer compress(DataBuffer buffer) { try { ByteArrayOutputStream stream = new ByteArrayOutputStream(); GZIPOutputStream gzip = new GZIPOutputStream(stream); DataOutputStream dos = new DataOutputStream(gzip); buffer.write(dos); dos.flush(); dos.close(); byte[] bytes = stream.toByteArray(); // logger.info("Bytes: {}", Arrays.toString(bytes)); BytePointer pointer = new BytePointer(bytes); CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); descriptor.setCompressedLength(bytes.length); CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor); return result; } catch (Exception e) { throw new RuntimeException(e); } }
@Override public DataBuffer compress(DataBuffer buffer) { CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); BytePointer ptr = new BytePointer(buffer.length() * buffer.getElementSize()); CompressedDataBuffer result = new CompressedDataBuffer(ptr, descriptor); Nd4j.getMemoryManager().memcpy(result, buffer); return result; }
@Override public DataBuffer decompress(DataBuffer buffer) { try { CompressedDataBuffer compressed = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); BytePointer pointer = (BytePointer) compressed.addressPointer(); ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes()); GZIPInputStream gzip = new GZIPInputStream(bis); DataInputStream dis = new DataInputStream(gzip); DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements()); bufferRestored.read(dis); return bufferRestored; } catch (Exception e) { throw new RuntimeException(e); } }
public CompressedDataBuffer(Pointer pointer, @NonNull CompressionDescriptor descriptor) { this.compressionDescriptor = descriptor; this.pointer = pointer; this.length = descriptor.getNumberOfElements(); this.elementSize = (byte) descriptor.getOriginalElementSize(); initTypeAndSize(); }
/** * * @param array * @return */ public INDArray decompress(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return array; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array); }