@Override public void write(DataOutputStream out) throws IOException { if (length() >= Integer.MAX_VALUE) throw new IllegalArgumentException("Length of data buffer can not be >= Integer.MAX_VALUE on output"); // log.info("Saving dType: {}", dataType().name()); out.writeUTF(allocationMode.name()); out.writeLong(length()); out.writeUTF(dataType().name()); if (dataType() == Type.DOUBLE) { for (int i = 0; i < length(); i++) out.writeDouble(getDouble(i)); } else if (dataType() == Type.LONG) { for (int i = 0; i < length(); i++) out.writeLong(getLong(i)); } else if (dataType() == Type.INT) { for (int i = 0; i < length(); i++) out.writeInt(getInt(i)); } else if (dataType() == Type.HALF) { for (int i = 0; i < length(); i++) { out.writeShort(getShort(i)); } } else { for (int i = 0; i < length(); i++) { out.writeFloat(getFloat(i)); } } }
allocated.putInt(arr.rank()); allocated.putInt(arr.data().dataType().ordinal());
length = locLength; Type t = Type.valueOf(s.readUTF()); if (t != globalType && (!t.equals(Type.INT) && !t.equals(Type.LONG)) && Nd4j.sizeOfDataType(globalType) < Nd4j.sizeOfDataType(t)) { log.warn("Loading a data stream with opType different from what is set globally. Expect precision loss"); if (globalType == Type.INT) throw new IllegalStateException("Unknown dataType: [" + t.toString() + "]");
String nd4jDataTypeName = DataTypeUtil.getDtypeFromContext().name();
DataBuffer.Type type = DataBuffer.Type.values()[byteBuffer.getInt()]; for (int i = 0; i < shapeBufferLength; i++) { shapeBuff.put(i, byteBuffer.getInt());
DataBuffer.Type type = data.dataType(); out.writeLong(array.length()); out.writeInt(type.ordinal()); switch (type) { case DOUBLE: {
allocated.putInt(arr.rank()); allocated.putInt(arr.data().dataType().ordinal());
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { ByteBuffer buffer1 = ByteBuffer.wrap(b1, s1, l1); ByteBuffer buffer2 = ByteBuffer.wrap(b2, s2, l2); long length1 = buffer1.getLong(); long length2 = buffer2.getLong(); if (length1 == 0 && length2 == 0) { return 0; } else if (length1 == 0) { return (int)Math.max(-length2, Integer.MIN_VALUE); } else if (length2 == 0) { return (int)Math.min(length1, Integer.MAX_VALUE); } int type1 = buffer1.getInt(); int type2 = buffer2.getInt(); if (type1 != type2) { throw new IllegalArgumentException("Data types must be the same."); } if (type1 == DataBuffer.Type.DOUBLE.ordinal()) { return buffer1.asDoubleBuffer().compareTo(buffer2.asDoubleBuffer()); } else if (type1 == DataBuffer.Type.FLOAT.ordinal()) { return buffer1.asFloatBuffer().compareTo(buffer2.asFloatBuffer()); } else if (type1 == DataBuffer.Type.INT.ordinal()) { return buffer1.asIntBuffer().compareTo(buffer2.asIntBuffer()); } else { throw new UnsupportedOperationException("Unsupported data type: " + type1); } } }
/** Deserialize into a row vector of default type. */ public void readFields(DataInput in) throws IOException { long length = in.readLong(); if (length == 0) { array = null; return; } int type = in.readInt(); if (array == null || array.length() != length) { if (length >= Integer.MAX_VALUE) { throw new IllegalArgumentException("Length can not be >= Integer.MAX_VALUE"); } array = Nd4j.zeros((int)length); } if (type == DataBuffer.Type.DOUBLE.ordinal()) { for (int i = 0; i < length; i++) { array.putScalar(i, in.readDouble()); } } else if (type == DataBuffer.Type.FLOAT.ordinal()) { for (int i = 0; i < length; i++) { array.putScalar(i, in.readFloat()); } } else if (type == DataBuffer.Type.INT.ordinal()) { for (int i = 0; i < length; i++) { array.putScalar(i, in.readInt()); } } else { throw new UnsupportedOperationException("Unsupported data type: " + type); } }
/** * Setup the given byte buffer * for serialization (note that this is for uncompressed INDArrays) * 4 bytes int for rank * 4 bytes for data type * shape buffer * data buffer * * @param arr the array to setup * @param allocated the byte buffer to setup * @param rewind whether to rewind the byte buffer or nt */ public static void doByteBufferPutUnCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) { // ensure we send data to host memory Nd4j.getExecutioner().commit(); Nd4j.getAffinityManager().ensureLocation(arr, AffinityManager.Location.HOST); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); //2 four byte ints at the beginning allocated.putInt(arr.rank()); //put data type next so its self describing allocated.putInt(arr.data().dataType().ordinal()); allocated.put(shapeBuffer); allocated.put(buffer); if (rewind) allocated.rewind(); }
@Override public void write(DataOutputStream out) throws IOException { // logger.info("Writing out CompressedDataBuffer"); // here we should mimic to usual DataBuffer array out.writeUTF(allocationMode.name()); out.writeInt((int) compressionDescriptor.getCompressedLength()); out.writeUTF(Type.COMPRESSED.name()); // at this moment we don't care about mimics anymore //ByteRawIndexer indexer = new ByteRawIndexer((BytePointer) pointer); out.writeUTF(compressionDescriptor.getCompressionAlgorithm()); out.writeLong(compressionDescriptor.getCompressedLength()); out.writeLong(compressionDescriptor.getOriginalLength()); out.writeLong(compressionDescriptor.getNumberOfElements()); // out.write(((BytePointer) pointer).getStringBytes()); for (int x = 0; x < pointer.capacity() * pointer.sizeof(); x++) { byte b = pointer.asByteBuffer().get(x); out.writeByte(b); } }
@Override public DataBuffer create(DataBuffer underlyingBuffer, long offset, long length) { if (underlyingBuffer.dataType() == DataBuffer.Type.DOUBLE) { return new CudaDoubleDataBuffer(underlyingBuffer, length, offset); } else if (underlyingBuffer.dataType() == DataBuffer.Type.FLOAT) { return new CudaFloatDataBuffer(underlyingBuffer, length, offset); } else if (underlyingBuffer.dataType() == DataBuffer.Type.INT) { return new CudaIntDataBuffer(underlyingBuffer, length, offset); } else if (underlyingBuffer.dataType() == DataBuffer.Type.HALF) { return new CudaHalfDataBuffer(underlyingBuffer, length, offset); } else if (underlyingBuffer.dataType() == DataBuffer.Type.LONG) { return new CudaLongDataBuffer(underlyingBuffer, length, offset); } throw new ND4JIllegalStateException("Unknown data buffer type: " + underlyingBuffer.dataType().toString()); }
DataBuffer.Type type = DataBuffer.Type.values()[byteBuffer.getInt()]; for (int i = 0; i < shapeBufferLength; i++) { shapeBuff.put(i, byteBuffer.getInt());
/** * Setup the given byte buffer * for serialization (note that this is for uncompressed INDArrays) * 4 bytes int for rank * 4 bytes for data opType * shape buffer * data buffer * * @param arr the array to setup * @param allocated the byte buffer to setup * @param rewind whether to rewind the byte buffer or nt */ public static void doByteBufferPutUnCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) { // ensure we send data to host memory Nd4j.getExecutioner().commit(); Nd4j.getAffinityManager().ensureLocation(arr, AffinityManager.Location.HOST); ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder()); //2 four byte ints at the beginning allocated.putInt(arr.rank()); //put data opType next so its self describing allocated.putInt(arr.data().dataType().ordinal()); allocated.put(shapeBuffer); allocated.put(buffer); if (rewind) allocated.rewind(); }
@Override public void write(DataOutputStream out) throws IOException { // logger.info("Writing out CompressedDataBuffer"); // here we should mimic to usual DataBuffer array out.writeUTF(allocationMode.name()); out.writeInt((int) compressionDescriptor.getCompressedLength()); out.writeUTF(Type.COMPRESSED.name()); // at this moment we don't care about mimics anymore //ByteRawIndexer indexer = new ByteRawIndexer((BytePointer) pointer); out.writeUTF(compressionDescriptor.getCompressionAlgorithm()); out.writeLong(compressionDescriptor.getCompressedLength()); out.writeLong(compressionDescriptor.getOriginalLength()); out.writeLong(compressionDescriptor.getNumberOfElements()); // out.write(((BytePointer) pointer).getStringBytes()); for (int x = 0; x < pointer.capacity() * pointer.sizeof(); x++) { byte b = pointer.asByteBuffer().get(x); out.writeByte(b); } }
static String type(INDArray arr) { return arr.data().dataType().name(); }
@Override public int hashCode() { if (intArray != null) { return intArray.getClass().hashCode() + 31 * Arrays.hashCode(intArray) + 31 * bufferType.ordinal(); } else if (floatArray != null) { return floatArray.getClass().hashCode() + 31 * Arrays.hashCode(floatArray) + 31 * bufferType.ordinal(); } else if (doubleArray != null) { return doubleArray.getClass().hashCode() + 31 * Arrays.hashCode(doubleArray) + 31 * bufferType.ordinal(); } else if (longArray != null) { return longArray.getClass().hashCode() + 31 * Arrays.hashCode(longArray) + 31 * bufferType.ordinal(); } else { return 0; } } }
static String arrayInfo(INDArray arr) { return arr.data().dataType().toString(); }