/** * In place compression of the passed in ndarray * with the default compression algorithm * @param array */ public void compressi(INDArray array) { compressi(array, getDefaultCompression()); }
/** * This method returns BasicNDArrayCompressor instance, * suitable for NDArray compression/decompression * at runtime * * @return */ public static BasicNDArrayCompressor getCompressor() { return BasicNDArrayCompressor.getInstance(); }
/** * Decompress several ndarrays * @param arrays */ public void autoDecompress(INDArray... arrays) { for (INDArray array : arrays) { autoDecompress(array); } }
/** * Instantiate a compression descriptor from * the given bytebuffer * @param byteBuffer the bytebuffer to instantiate * the descriptor from * @return the instantiated descriptor based on the given * bytebuffer */ public static CompressionDescriptor fromByteBuffer(ByteBuffer byteBuffer) { CompressionDescriptor compressionDescriptor = new CompressionDescriptor(); //compression opType int compressionTypeOrdinal = byteBuffer.getInt(); CompressionType compressionType = CompressionType.values()[compressionTypeOrdinal]; compressionDescriptor.setCompressionType(compressionType); //compression algo int compressionAlgoOrdinal = byteBuffer.getInt(); CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.values()[compressionAlgoOrdinal]; compressionDescriptor.setCompressionAlgorithm(compressionAlgorithm.name()); //from here everything is longs compressionDescriptor.setOriginalLength(byteBuffer.getLong()); compressionDescriptor.setCompressedLength(byteBuffer.getLong()); compressionDescriptor.setNumberOfElements(byteBuffer.getLong()); compressionDescriptor.setOriginalElementSize(byteBuffer.getLong()); return compressionDescriptor; }
/** * Decompress the given databuffer * @param buffer the databuffer to compress * @return the decompressed databuffer */ public DataBuffer decompress(DataBuffer buffer) { if (buffer.dataType() != DataBuffer.Type.COMPRESSED) throw new IllegalStateException("You can't decompress DataBuffer with dataType of: " + buffer.dataType()); CompressedDataBuffer comp = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(buffer); }
/** * in place decompression of the given * ndarray. If the ndarray isn't compressed * this will do nothing * @param array the array to decompressed * if it is comprssed */ public void decompressi(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); codecs.get(descriptor.getCompressionAlgorithm()).decompressi(array); }
@Override public DataBuffer compress(DataBuffer buffer) { CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); BytePointer ptr = new BytePointer(buffer.length() * buffer.getElementSize()); CompressedDataBuffer result = new CompressedDataBuffer(ptr, descriptor); Nd4j.getMemoryManager().memcpy(result, buffer); return result; }
@Override public DataBuffer dup() { Pointer nPtr = new BytePointer(compressionDescriptor.getCompressedLength()); val perfD = PerformanceTracker.getInstance().helperStartTransaction(); Pointer.memcpy(nPtr, pointer, compressionDescriptor.getCompressedLength()); PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, compressionDescriptor.getCompressedLength(), MemcpyDirection.HOST_TO_HOST); CompressionDescriptor nDesc = compressionDescriptor.clone(); CompressedDataBuffer nBuf = new CompressedDataBuffer(nPtr, nDesc); return nBuf; }
public CompressedDataBuffer(Pointer pointer, @NonNull CompressionDescriptor descriptor) { this.compressionDescriptor = descriptor; this.pointer = pointer; this.length = descriptor.getNumberOfElements(); this.elementSize = (byte) descriptor.getOriginalElementSize(); initTypeAndSize(); }
@Override public CompressionDescriptor clone() { CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.compressionType = this.compressionType; descriptor.compressionAlgorithm = this.compressionAlgorithm; descriptor.originalLength = this.originalLength; descriptor.compressedLength = this.compressedLength; descriptor.numberOfElements = this.numberOfElements; descriptor.originalElementSize = this.originalElementSize; return descriptor; } }
@Override public long length() { return compressionDescriptor.getNumberOfElements(); }
/** * * @param array */ public void autoDecompress(INDArray array) { if (array.isCompressed()) decompressi(array); }
/** * This method returns compressed INDArray instance which contains JVM array passed in * * @param array * @return */ public INDArray compress(float[] array) { return codecs.get(defaultCompression).compress(array); }
/** * Compress the given data buffer * given the default compression algorithm * @param buffer the data buffer to compress * @return the compressed version of the data buffer */ public DataBuffer compress(DataBuffer buffer) { return compress(buffer, getDefaultCompression()); }
/** * Initialize a compression descriptor * based on the given data buffer (for the sizes) * and the compressor to get the opType * @param buffer * @param compressor */ public CompressionDescriptor(DataBuffer buffer, NDArrayCompressor compressor) { this(buffer, compressor.getDescriptor()); this.compressionType = compressor.getCompressionType(); }
private BasicNDArrayCompressor() { loadCompressors(); }
public static boolean goingToCompress(@NonNull DataBuffer.TypeEx from, @NonNull DataBuffer.TypeEx to) { if (!goingToDecompress(from, to) && goingToDecompress(to, from)) return true; return false; } }
@Override public DataBuffer decompress(DataBuffer buffer) { CompressedDataBuffer comp = (CompressedDataBuffer) buffer; DataBuffer result = Nd4j.createBuffer(comp.length(), false); Nd4j.getMemoryManager().memcpy(result, buffer); return result; }
/** * * @param array * @return */ public INDArray decompress(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return array; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array); }
/** * This method returns compressed INDArray instance which contains JVM array passed in * * @param array * @return */ public INDArray compress(double[] array) { return codecs.get(defaultCompression).compress(array); } }