/** * Instantiate a compression descriptor from * the given bytebuffer * @param byteBuffer the bytebuffer to instantiate * the descriptor from * @return the instantiated descriptor based on the given * bytebuffer */ public static CompressionDescriptor fromByteBuffer(ByteBuffer byteBuffer) { CompressionDescriptor compressionDescriptor = new CompressionDescriptor(); //compression opType int compressionTypeOrdinal = byteBuffer.getInt(); CompressionType compressionType = CompressionType.values()[compressionTypeOrdinal]; compressionDescriptor.setCompressionType(compressionType); //compression algo int compressionAlgoOrdinal = byteBuffer.getInt(); CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.values()[compressionAlgoOrdinal]; compressionDescriptor.setCompressionAlgorithm(compressionAlgorithm.name()); //from here everything is longs compressionDescriptor.setOriginalLength(byteBuffer.getLong()); compressionDescriptor.setCompressedLength(byteBuffer.getLong()); compressionDescriptor.setNumberOfElements(byteBuffer.getLong()); compressionDescriptor.setOriginalElementSize(byteBuffer.getLong()); return compressionDescriptor; }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.UINT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT16, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressionType(getCompressionType()); descriptor.setOriginalLength(length * elementSize); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setOriginalElementSize(elementSize); descriptor.setCompressedLength(length * elementSize); descriptor.setNumberOfElements(length); BytePointer ptr = new BytePointer(length * elementSize); val perfD = PerformanceTracker.getInstance().helperStartTransaction(); // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS Pointer.memcpy(ptr, srcPointer, length * elementSize); PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, length * elementSize, MemcpyDirection.HOST_TO_HOST); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); return buffer; } }
/** * Instantiate a compression descriptor from * the given bytebuffer * @param byteBuffer the bytebuffer to instantiate * the descriptor from * @return the instantiated descriptor based on the given * bytebuffer */ public static CompressionDescriptor fromByteBuffer(ByteBuffer byteBuffer) { CompressionDescriptor compressionDescriptor = new CompressionDescriptor(); //compression type int compressionTypeOrdinal = byteBuffer.getInt(); CompressionType compressionType = CompressionType.values()[compressionTypeOrdinal]; compressionDescriptor.setCompressionType(compressionType); //compression algo int compressionAlgoOrdinal = byteBuffer.getInt(); CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.values()[compressionAlgoOrdinal]; compressionDescriptor.setCompressionAlgorithm(compressionAlgorithm.name()); //from here everything is longs compressionDescriptor.setOriginalLength(byteBuffer.getLong()); compressionDescriptor.setCompressedLength(byteBuffer.getLong()); compressionDescriptor.setNumberOfElements(byteBuffer.getLong()); compressionDescriptor.setOriginalElementSize(byteBuffer.getLong()); return compressionDescriptor; }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressionType(getCompressionType()); descriptor.setOriginalLength(length * elementSize); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setOriginalElementSize(elementSize); descriptor.setCompressedLength(length * elementSize); descriptor.setNumberOfElements(length); BytePointer ptr = new BytePointer(length * elementSize); // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS Pointer.memcpy(ptr, srcPointer, length * elementSize); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT16, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.UINT8, ptr, length); return buffer; } }
descriptor.setCompressionType(getCompressionType());
descriptor.setCompressionType(getCompressionType());
descriptor.setCompressionType(CompressionType.LOSSY); descriptor.setCompressedLength(source.length() * elementSize); buffer = new CompressedDataBuffer(pointer, descriptor);
descriptor.setCompressionType(CompressionType.LOSSY); descriptor.setCompressedLength(source.length() * elementSize); buffer = new CompressedDataBuffer(pointer, descriptor);