/** * Pass in an ndarray to get the databuffer * and the appropriate dimensions * @param array the array to pass in * to get the shape info from * @param dimension the dimensions for the TAD */ public TadDescriptor(INDArray array, int[] dimension) { this.dimensionLength = dimension == null ? 0 : dimension.length; this.dimension = dimension; // TODO: change this to fill shapeInfo this.shape = dataBufferToArray(array.shapeInfoDataBuffer()); }
public abstract DataBuffer decompress(DataBuffer buffer);
if (!arr.isCompressed()) { ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); ByteBuffer codecByteBuffer = descriptor.toByteBuffer(); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); int twoInts = 2 * 4; return twoInts + buffer.limit() + shapeBuffer.limit() + codecByteBuffer.limit();
ByteBuffer codecByteBuffer = descriptor.toByteBuffer(); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); allocated.putInt(arr.rank());
/** * Setup the given byte buffer * for serialization (note that this is for uncompressed INDArrays) * 4 bytes int for rank * 4 bytes for data opType * shape buffer * data buffer * * @param arr the array to setup * @param allocated the byte buffer to setup * @param rewind whether to rewind the byte buffer or nt */ public static void doByteBufferPutUnCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) { // ensure we send data to host memory Nd4j.getExecutioner().commit(); Nd4j.getAffinityManager().ensureLocation(arr, AffinityManager.Location.HOST); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); //2 four byte ints at the beginning allocated.putInt(arr.rank()); //put data opType next so its self describing allocated.putInt(arr.data().dataType().ordinal()); allocated.put(shapeBuffer); allocated.put(buffer); if (rewind) allocated.rewind(); }
.append(Arrays.toString(ndarray.shapeInfoDataBuffer().asInt())).append("; Values: ").append(Arrays.toString(ndarray.data().asFloat())).append(";\n");
@Override public INDArray bitmapEncode(INDArray indArray, double threshold) { DataBuffer buffer = Nd4j.getDataBufferFactory().createInt(indArray.length() / 16 + 5); INDArray ret = Nd4j.createArrayFromShapeBuffer(buffer, indArray.shapeInfoDataBuffer()); bitmapEncode(indArray, ret, threshold); return ret; }
/** * Write an ndarray to the specified outputstream * * @param arr the array to write * @param dataOutputStream the data output stream to write to * @throws IOException */ public static void write(INDArray arr, DataOutputStream dataOutputStream) throws IOException { //BaseDataBuffer.write(...) doesn't know about strides etc, so dup (or equiv. strategy) is necessary here //Furthermore, because we only want to save the *actual* data for a view (not the full data), the shape info // (mainly strides, offset, element-wise stride) may be different in the duped array vs. the view array if (arr.isView()) arr = arr.dup(); arr.shapeInfoDataBuffer().write(dataOutputStream); arr.data().write(dataOutputStream); }
indexes = NDArrayIndex.resolve(arr.shapeInfoDataBuffer(), indexes); if (tryShortCircuit(indexes)) { return;
protected void write(ObjectOutputStream out) throws IOException { if (this.isView()) { //As per Nd4j.write, duplicate before writing to the output stream //BaseDataBuffer.write(...) doesn't know about strides etc, so dup (or equiv. strategy) is necessary here //Furthermore, because we only want to save the *actual* data for a view (not the full data), the shape info // (mainly strides, offset, element-wise stride) may be different in the duped array vs. the view array INDArray copy = this.dup(); copy.shapeInfoDataBuffer().write(out); copy.data().write(out); } else { shapeInformation.write(out); data().write(out); } }
/** * Pass in an ndarray to get the databuffer * and the appropriate dimensions * @param array the array to pass in * to get the shape info from * @param dimension the dimensions for the TAD */ public TadDescriptor(INDArray array, int[] dimension) { this.dimensionLength = dimension == null ? 0 : dimension.length; this.dimension = dimension; // TODO: change this to fill shapeInfo this.shape = dataBufferToArray(array.shapeInfoDataBuffer()); }
/** * Utility method, to associate INDArray with specific device (backend-specific) * * @param array */ @Override public void touch(INDArray array) { if (array == null) return; touch(array.data()); touch(array.shapeInfoDataBuffer()); }
/** * Utility method, to associate INDArray with specific device (backend-specific) * * @param array */ @Override public void touch(INDArray array) { if (array == null) return; touch(array.data()); touch(array.shapeInfoDataBuffer()); }
protected void prepareDelayedMemory(INDArray array) { if (configuration.getMemoryModel() == Configuration.MemoryModel.DELAYED) { AllocationPoint pointData = allocator.getAllocationPoint(array.shapeInfoDataBuffer()); AllocationPoint pointShape = allocator.getAllocationPoint(array.shapeInfoDataBuffer()); if (pointData.getAllocationStatus() != AllocationStatus.DEVICE) prepareDelayedMemory(array.data()); if (pointShape.getAllocationStatus() == AllocationStatus.HOST) { DataBuffer oShape = array.shapeInfoDataBuffer(); DataBuffer nShape = Nd4j.getConstantHandler().relocateConstantSpace(oShape); if (nShape == oShape) Nd4j.getConstantHandler().moveToConstantSpace(nShape); ((JCublasNDArray) array).setShapeInfoDataBuffer(nShape); } } }
protected void prepareDelayedMemory(INDArray array) { if (configuration.getMemoryModel() == Configuration.MemoryModel.DELAYED) { AllocationPoint pointData = allocator.getAllocationPoint(array.shapeInfoDataBuffer()); AllocationPoint pointShape = allocator.getAllocationPoint(array.shapeInfoDataBuffer()); if (pointData.getAllocationStatus() != AllocationStatus.DEVICE) prepareDelayedMemory(array.data()); if (pointShape.getAllocationStatus() == AllocationStatus.HOST) { DataBuffer oShape = array.shapeInfoDataBuffer(); DataBuffer nShape = Nd4j.getConstantHandler().relocateConstantSpace(oShape); if (nShape == oShape) Nd4j.getConstantHandler().moveToConstantSpace(nShape); ((JCublasNDArray) array).setShapeInfoDataBuffer(nShape); } } }
@Override public INDArray bitmapEncode(INDArray indArray, double threshold) { DataBuffer buffer = Nd4j.getDataBufferFactory().createInt(indArray.length() / 16 + 5); INDArray ret = Nd4j.createArrayFromShapeBuffer(buffer, indArray.shapeInfoDataBuffer()); bitmapEncode(indArray, ret, threshold); return ret; }
/** * Write an ndarray to the specified outputstream * * @param arr the array to write * @param dataOutputStream the data output stream to write to * @throws IOException */ public static void write(INDArray arr, DataOutputStream dataOutputStream) throws IOException { //BaseDataBuffer.write(...) doesn't know about strides etc, so dup (or equiv. strategy) is necessary here //Furthermore, because we only want to save the *actual* data for a view (not the full data), the shape info // (mainly strides, offset, element-wise stride) may be different in the duped array vs. the view array if (arr.isView()) arr = arr.dup(); arr.shapeInfoDataBuffer().write(dataOutputStream); arr.data().write(dataOutputStream); }
@Override public INDArray compress(INDArray array) { //logger.info("Threshold [{}] compression", threshold); Nd4j.getExecutioner().commit(); //Nd4j.getAffinityManager().ensureLocation(array, AffinityManager.Location.HOST); DataBuffer buffer = compress(array.data()); if (buffer == null) return null; INDArray dup = Nd4j.createArrayFromShapeBuffer(buffer, array.shapeInfoDataBuffer()); dup.markAsCompressed(true); return dup; }
@Override public INDArray compress(INDArray array) { //logger.info("Threshold [{}] compression", threshold); Nd4j.getExecutioner().commit(); //Nd4j.getAffinityManager().ensureLocation(array, AffinityManager.Location.HOST); DataBuffer buffer = compress(array.data()); if (buffer == null) return null; INDArray dup = Nd4j.createArrayFromShapeBuffer(buffer, array.shapeInfoDataBuffer()); dup.markAsCompressed(true); return dup; }
protected void write(ObjectOutputStream out) throws IOException { if (this.isView()) { //As per Nd4j.write, duplicate before writing to the output stream //BaseDataBuffer.write(...) doesn't know about strides etc, so dup (or equiv. strategy) is necessary here //Furthermore, because we only want to save the *actual* data for a view (not the full data), the shape info // (mainly strides, offset, element-wise stride) may be different in the duped array vs. the view array INDArray copy = this.dup(); copy.shapeInfoDataBuffer().write(out); copy.data().write(out); } else { shapeInformation.write(out); data().write(out); } }