/** * Decompress the given databuffer * @param buffer the databuffer to compress * @return the decompressed databuffer */ public DataBuffer decompress(DataBuffer buffer) { if (buffer.dataType() != DataBuffer.Type.COMPRESSED) throw new IllegalStateException("You can't decompress DataBuffer with dataType of: " + buffer.dataType()); CompressedDataBuffer comp = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(buffer); }
/** * * @param array * @return */ public INDArray decompress(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return array; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array); }
/** * in place decompression of the given * ndarray. If the ndarray isn't compressed * this will do nothing * @param array the array to decompressed * if it is comprssed */ public void decompressi(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); codecs.get(descriptor.getCompressionAlgorithm()).decompressi(array); }
@Override public void write(DataOutputStream out) throws IOException { // logger.info("Writing out CompressedDataBuffer"); // here we should mimic to usual DataBuffer array out.writeUTF(allocationMode.name()); out.writeInt((int) compressionDescriptor.getCompressedLength()); out.writeUTF(Type.COMPRESSED.name()); // at this moment we don't care about mimics anymore //ByteRawIndexer indexer = new ByteRawIndexer((BytePointer) pointer); out.writeUTF(compressionDescriptor.getCompressionAlgorithm()); out.writeLong(compressionDescriptor.getCompressedLength()); out.writeLong(compressionDescriptor.getOriginalLength()); out.writeLong(compressionDescriptor.getNumberOfElements()); // out.write(((BytePointer) pointer).getStringBytes()); for (int x = 0; x < pointer.capacity() * pointer.sizeof(); x++) { byte b = pointer.asByteBuffer().get(x); out.writeByte(b); } }
/** * Decompress the given databuffer * @param buffer the databuffer to compress * @return the decompressed databuffer */ public DataBuffer decompress(DataBuffer buffer) { if (buffer.dataType() != DataBuffer.Type.COMPRESSED) throw new IllegalStateException("You can't decompress DataBuffer with dataType of: " + buffer.dataType()); CompressedDataBuffer comp = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(buffer); }
/** * * @param array * @return */ public INDArray decompress(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return array; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array); }
/** * in place decompression of the given * ndarray. If the ndarray isn't compressed * this will do nothing * @param array the array to decompressed * if it is comprssed */ public void decompressi(INDArray array) { if (array.data().dataType() != DataBuffer.Type.COMPRESSED) return; CompressedDataBuffer comp = (CompressedDataBuffer) array.data(); CompressionDescriptor descriptor = comp.getCompressionDescriptor(); if (!codecs.containsKey(descriptor.getCompressionAlgorithm())) throw new RuntimeException("Non-existent compression algorithm requested: [" + descriptor.getCompressionAlgorithm() + "]"); codecs.get(descriptor.getCompressionAlgorithm()).decompressi(array); }
@Override public void write(DataOutputStream out) throws IOException { // logger.info("Writing out CompressedDataBuffer"); // here we should mimic to usual DataBuffer array out.writeUTF(allocationMode.name()); out.writeInt((int) compressionDescriptor.getCompressedLength()); out.writeUTF(Type.COMPRESSED.name()); // at this moment we don't care about mimics anymore //ByteRawIndexer indexer = new ByteRawIndexer((BytePointer) pointer); out.writeUTF(compressionDescriptor.getCompressionAlgorithm()); out.writeLong(compressionDescriptor.getCompressedLength()); out.writeLong(compressionDescriptor.getOriginalLength()); out.writeLong(compressionDescriptor.getNumberOfElements()); // out.write(((BytePointer) pointer).getStringBytes()); for (int x = 0; x < pointer.capacity() * pointer.sizeof(); x++) { byte b = pointer.asByteBuffer().get(x); out.writeByte(b); } }
CompressedDataBuffer compressedData = (CompressedDataBuffer)data; CompressionDescriptor desc = compressedData.getCompressionDescriptor(); String algo = desc.getCompressionAlgorithm(); switch (algo) { case "FLOAT16": type = DT_HALF; break;