protected double getPercentile(Number quantile, INDArray sorted) { if (quantile.intValue() == 0) return sorted.getDouble(0); else if (quantile.intValue() == 100) return sorted.getDouble(sorted.length() - 1); double pos = (quantile.doubleValue() / 100.0) * (double) (sorted.length() + 1); double fposition = FastMath.floor(pos); int position = (int)fposition; double diff = pos - fposition; double lower = sorted.getDouble(position-1); double upper = sorted.getDouble(position); return lower + diff * (upper - lower); }
public LogEntry(INDArray toLog, String status) { //this.id = toLog.id(); this.shape = toLog.shape(); this.stride = toLog.stride(); this.ndArrayType = toLog.getClass().getName(); this.length = toLog.length(); this.references = toLog.data().references(); this.dataType = toLog.data().dataType() == DataBuffer.Type.DOUBLE ? "double" : "float"; this.timestamp = System.currentTimeMillis(); this.stackTraceElements = Thread.currentThread().getStackTrace(); this.status = status; }
/** * Returns the double data * for this ndarray. * If possible (the offset is 0 representing the whole buffer) * it will return a direct reference to the underlying array * @param buf the ndarray to get the data for * @return the double data for this ndarray */ public static double[] getDoubleData(INDArray buf) { if (buf.data().dataType() != DataBuffer.Type.DOUBLE) throw new IllegalArgumentException("Double data must be obtained from a double buffer"); if (buf.data().allocationMode() == DataBuffer.AllocationMode.HEAP) { return buf.data().asDouble(); } else { double[] ret = new double[(int) buf.length()]; INDArray linear = buf.linearView(); for (int i = 0; i < buf.length(); i++) ret[i] = linear.getDouble(i); return ret; } }
@Override public INDArray percentile(Number quantile, int... dimension) { if (quantile.doubleValue() < 0 || quantile.doubleValue() > 100) throw new ND4JIllegalStateException("Percentile value should be in 0...100 range"); if (isScalar()) return Nd4j.scalar(this.getDouble(0)); INDArray sorted = Nd4j.getNDArrayFactory().sort(this.dup(this.ordering()), false, dimension); // there's no practical sense doing this on GPU, stride will be just size of TAD. INDArray ret = Nd4j.createUninitialized(sorted.tensorssAlongDimension(dimension)); for (int i = 0; i < ret.length(); i++) { ret.putScalar(i, getPercentile(quantile, sorted.tensorAlongDimension(i, dimension))); } return ret; }
for (int i = 0; i < retLinear.length(); i++) retLinear.putScalar(i, toConcat[i].getDouble(0)); return ret; int currBuffer = 0; int currBufferOffset = 0; for (int i = 0; i < ret.length(); i++) { ret.data().put(i, toConcat[currBuffer].data() .getDouble(toConcat[currBuffer].offset() + currBufferOffset++)); if (currBufferOffset >= toConcat[currBuffer].length()) { currBuffer++; currBufferOffset = 0; retAlongDimensionArrays[i] = ret.tensorAlongDimension(i, dimension); INDArray arrTensor = arr.tensorAlongDimension(i, dimension); arrTensorLength = arrTensor.length(); for (int j = 0; j < arrTensor.length(); j++) { int idx = j + arrOffset; retLinear.putScalar(idx, arrTensor.getDouble(j));
@Override public INDArray repeat(int dimension, long... repeats) { Nd4j.getCompressor().autoDecompress(this); if (dimension < 0) dimension += rank(); if (repeats.length < rank()) { if (dimension > 0) repeats = Longs.concat(ArrayUtil.nTimes((long) rank() - repeats.length, 1), repeats); //append rather than prepend for dimension == 0 else repeats = Longs.concat(repeats, ArrayUtil.nTimes((long) rank() - repeats.length, 1)); } long[] newShape = new long[rank()]; for (int i = 0; i < newShape.length; i++) newShape[i] = size(i) * repeats[i]; INDArray ret = Nd4j.create(newShape); //number of times to repeat each value long repeatDelta = ArrayUtil.prod(newShape) / length(); for (int i = 0; i < tensorssAlongDimension(dimension); i++) { INDArray thisTensor = tensorAlongDimension(i, dimension); INDArray retTensor = ret.tensorAlongDimension(i, dimension); int retIdx = 0; for (int k = 0; k < thisTensor.length(); k++) { for (int j = 0; j < repeatDelta; j++) { retTensor.putScalar(retIdx++, thisTensor.getDouble(k)); } } } return ret; }
INDArray as2d; if (shape[0] == 1) { as2d = data.tensorAlongDimension(0, 1, 2).permutei(1, 0); //Edge case: miniBatchSize==1 } else if (shape[2] == 1) { as2d = data.tensorAlongDimension(0, 1, 0); //Edge case: timeSeriesLength=1 } else { INDArray permuted = data.permute(0, 2, 1); //Permute, so we get correct order after reshaping INDArray mask1d = mask.reshape('f', new long[] {mask.length(), 1}); if (numElements == mask.length()) { return as2d; //All are 1s float[] floatMask1d = mask1d.data().asFloat(); int currCount = 0; for (int i = 0; i < floatMask1d.length; i++) {
@Override public Op opForDimension(int index, int... dimension) { INDArray xAlongDimension = x.tensorAlongDimension(index, dimension); if (y() != null) return new Negative(xAlongDimension, y.tensorAlongDimension(index, dimension), z.tensorAlongDimension(index, dimension), xAlongDimension.length()); else return new Negative(xAlongDimension, z.tensorAlongDimension(index, dimension), x.lengthLong()); } }
protected double getPercentile(Number quantile, INDArray sorted) { if (quantile.intValue() == 0) return sorted.getDouble(0); else if (quantile.intValue() == 100) return sorted.getDouble(sorted.length() - 1); double pos = (quantile.doubleValue() / 100.0) * (double) (sorted.length() + 1); double fposition = FastMath.floor(pos); int position = (int) fposition; double diff = pos - fposition; double lower = sorted.getDouble(position - 1); double upper = sorted.getDouble(position); return lower + diff * (upper - lower); }
/** * Convert an ndarray to a byte array * @param arr the array to convert * @return the converted byte array * @throws IOException */ public static byte[] toByteArray(INDArray arr) throws IOException { if (arr.length() * arr.data().getElementSize() > Integer.MAX_VALUE) throw new ND4JIllegalStateException(""); ByteArrayOutputStream bos = new ByteArrayOutputStream((int) (arr.length() * arr.data().getElementSize())); DataOutputStream dos = new DataOutputStream(bos); write(arr, dos); byte[] ret = bos.toByteArray(); return ret; }
@Override public INDArray percentile(Number quantile, int... dimension) { if (quantile.doubleValue() < 0 || quantile.doubleValue() > 100) throw new ND4JIllegalStateException("Percentile value should be in 0...100 range"); if (isScalar()) return Nd4j.scalar(this.getDouble(0)); INDArray sorted = Nd4j.getNDArrayFactory().sort(this.dup(this.ordering()), false, dimension); // there's no practical sense doing this on GPU, stride will be just size of TAD. INDArray ret = Nd4j.createUninitialized(sorted.tensorssAlongDimension(dimension)); for (int i = 0; i < ret.length(); i++) { ret.putScalar(i, getPercentile(quantile, sorted.tensorAlongDimension(i, dimension))); } return ret; }
@Override public List<long[]> calculateOutputShape() { int numArgs = args().length; if(numArgs < 1) return Collections.emptyList(); val shape = args()[0].getArr(); if(shape == null) return Collections.emptyList(); else { if(shape.length() == 1) { if(shape.getDouble(0) < 1) return Arrays.asList(new long[]{1,1}); else return Arrays.asList(new long[]{1,shape.getInt(0)}); } } return Arrays.asList(shape.data().asLong()); }
@Override public Op opForDimension(int index, int... dimension) { INDArray xAlongDimension = x.tensorAlongDimension(index, dimension); if (y() != null) return new Floor(xAlongDimension, y.tensorAlongDimension(index, dimension), z.tensorAlongDimension(index, dimension), xAlongDimension.length()); else return new Floor(xAlongDimension, z.tensorAlongDimension(index, dimension), x.lengthLong()); } }
public LogEntry(INDArray toLog, StackTraceElement[] stackTraceElements, String status) { //this.id = toLog.id(); this.shape = toLog.shape(); this.stride = toLog.stride(); this.ndArrayType = toLog.getClass().getName(); this.length = toLog.length(); this.references = toLog.data().references(); this.dataType = toLog.data().dataType() == DataBuffer.Type.DOUBLE ? "double" : "float"; this.timestamp = System.currentTimeMillis(); this.stackTraceElements = stackTraceElements; this.status = status; }
@Override public Op opForDimension(int index, int... dimension) { INDArray xAlongDimension = x.tensorAlongDimension(index, dimension); return new Atan2Op(xAlongDimension, y.tensorAlongDimension(index, dimension), z.tensorAlongDimension(index, dimension), xAlongDimension.length()); }
if (arr.offset() < 1 && arr.data().length() == arr.length() || arr instanceof IComplexNDArray && arr.length() * 2 == arr.data().length()) if (arr.ordering() == 'f' && arr.stride(-1) != arr.elementStride() || arr.ordering() == 'c' && arr.stride(0) != arr.elementStride()) } else { INDArray ret = Nd4j.create(arr.shape()); for (int i = 0; i < ret.length(); i++) ret.putScalar(i, arr.getDouble(i)); return ret;
/** * Given a sequence of Iterators over a transform of matrices, fill in all of * the matrices with the entries in the theta vector. Errors are * thrown if the theta vector does not exactly fill the matrices. */ public static void setParams(INDArray theta, Collection<INDArray>... matrices) { int index = 0; for (Collection<INDArray> matrixCollection : matrices) { for (INDArray matrix : matrixCollection) { INDArray linear = matrix.linearView(); for (int i = 0; i < matrix.length(); i++) { linear.putScalar(i, theta.getDouble(index)); index++; } } } if (index != theta.length()) { throw new AssertionError("Did not entirely use the theta vector"); } }
public static void printNDArrayHeader(INDArray array) { System.out.println(array.data().dataType() + " - order=" + array.ordering() + ", offset=" + array.offset() + ", shape=" + Arrays.toString(array.shape()) + ", stride=" + Arrays.toString(array.stride()) + ", length=" + array.length() + ", data().length()=" + array.data().length()); }
@Override public Op opForDimension(int index, int... dimension) { INDArray xAlongDimension = x.tensorAlongDimension(index, dimension); if (y() != null) return new SoftSignDerivative(x.tensorAlongDimension(index, dimension), y.tensorAlongDimension(index, dimension), z.tensorAlongDimension(index, dimension), xAlongDimension.length()); else return new SoftSignDerivative(x.tensorAlongDimension(index, dimension), z.tensorAlongDimension(index, dimension), xAlongDimension.length()); }