@Override public DataBuffer compress(DataBuffer buffer) { try { ByteArrayOutputStream stream = new ByteArrayOutputStream(); GZIPOutputStream gzip = new GZIPOutputStream(stream); DataOutputStream dos = new DataOutputStream(gzip); buffer.write(dos); dos.flush(); dos.close(); byte[] bytes = stream.toByteArray(); // logger.info("Bytes: {}", Arrays.toString(bytes)); BytePointer pointer = new BytePointer(bytes); CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); descriptor.setCompressedLength(bytes.length); CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor); return result; } catch (Exception e) { throw new RuntimeException(e); } }
descriptor.setOriginalLength(originalLength); descriptor.setNumberOfElements(numberOfElements); return new CompressedDataBuffer(pointer, descriptor); } catch (Exception e) { throw new RuntimeException(e);
new CompressedDataBuffer(byteBufferPointer, compressionDescriptor);
@Override public DataBuffer compress(DataBuffer buffer) { CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); BytePointer ptr = new BytePointer(buffer.length() * buffer.getElementSize()); CompressedDataBuffer result = new CompressedDataBuffer(ptr, descriptor); Nd4j.getMemoryManager().memcpy(result, buffer); return result; }
@Override public DataBuffer dup() { Pointer nPtr = new BytePointer(compressionDescriptor.getCompressedLength()); val perfD = PerformanceTracker.getInstance().helperStartTransaction(); Pointer.memcpy(nPtr, pointer, compressionDescriptor.getCompressedLength()); PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, compressionDescriptor.getCompressedLength(), MemcpyDirection.HOST_TO_HOST); CompressionDescriptor nDesc = compressionDescriptor.clone(); CompressedDataBuffer nBuf = new CompressedDataBuffer(nPtr, nDesc); return nBuf; }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT16, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.UINT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressionType(getCompressionType()); descriptor.setOriginalLength(length * elementSize); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setOriginalElementSize(elementSize); descriptor.setCompressedLength(length * elementSize); descriptor.setNumberOfElements(length); BytePointer ptr = new BytePointer(length * elementSize); val perfD = PerformanceTracker.getInstance().helperStartTransaction(); // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS Pointer.memcpy(ptr, srcPointer, length * elementSize); PerformanceTracker.getInstance().helperRegisterTransaction(0, perfD, length * elementSize, MemcpyDirection.HOST_TO_HOST); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); return buffer; } }
@Override public DataBuffer compress(DataBuffer buffer) { try { ByteArrayOutputStream stream = new ByteArrayOutputStream(); GZIPOutputStream gzip = new GZIPOutputStream(stream); DataOutputStream dos = new DataOutputStream(gzip); buffer.write(dos); dos.flush(); dos.close(); byte[] bytes = stream.toByteArray(); // logger.info("Bytes: {}", Arrays.toString(bytes)); BytePointer pointer = new BytePointer(bytes); CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); descriptor.setCompressedLength(bytes.length); CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor); return result; } catch (Exception e) { throw new RuntimeException(e); } }
@Override public DataBuffer dup() { Pointer nPtr = new BytePointer(compressionDescriptor.getCompressedLength()); Pointer.memcpy(nPtr, pointer, compressionDescriptor.getCompressedLength()); CompressionDescriptor nDesc = compressionDescriptor.clone(); CompressedDataBuffer nBuf = new CompressedDataBuffer(nPtr, nDesc); return nBuf; }
@Override public DataBuffer compress(DataBuffer buffer) { CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this); BytePointer ptr = new BytePointer(buffer.length() * buffer.getElementSize()); CompressedDataBuffer result = new CompressedDataBuffer(ptr, descriptor); Nd4j.getMemoryManager().memcpy(result, buffer); return result; }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressionType(getCompressionType()); descriptor.setOriginalLength(length * elementSize); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setOriginalElementSize(elementSize); descriptor.setCompressedLength(length * elementSize); descriptor.setNumberOfElements(length); BytePointer ptr = new BytePointer(length * elementSize); // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS Pointer.memcpy(ptr, srcPointer, length * elementSize); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 1); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.UINT8, ptr, length); return buffer; } }
@Override protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) { BytePointer ptr = new BytePointer(length * 2); CompressionDescriptor descriptor = new CompressionDescriptor(); descriptor.setCompressedLength(length * 2); descriptor.setOriginalLength(length * elementSize); descriptor.setOriginalElementSize(elementSize); descriptor.setNumberOfElements(length); descriptor.setCompressionAlgorithm(getDescriptor()); descriptor.setCompressionType(getCompressionType()); CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor); Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT16, ptr, length); return buffer; } }