static INDArrayIndex[] createIntervalOnDimension(int dimension, boolean inclusive, long... interval) { INDArrayIndex [] indexInterval = new INDArrayIndex[dimension + 1]; for(int i = 0; i <= dimension; i++) { indexInterval[i] = i != dimension ? NDArrayIndex.all() : NDArrayIndex.interval((int)interval[0], (int)interval[1], inclusive); } return indexInterval; } }
/** * Generate an all index * equal to the rank of the given array * @param arr the array to generate the all index for * @return an ndarray index array containing of length * arr.rank() containing all elements */ public static INDArrayIndex[] allFor(INDArray arr) { INDArrayIndex[] ret = new INDArrayIndex[arr.rank()]; for (int i = 0; i < ret.length; i++) ret[i] = NDArrayIndex.all(); return ret; }
private static INDArray getSubsetForExample(INDArray array, int idx) { //Note the interval use here: normally .point(idx) would be used, but this collapses the point dimension // when used on arrays with rank of 3 or greater //So (point,all,all) on a 3d input returns a 2d output. Whereas, we want a 3d [1,x,y] output here switch (array.rank()) { case 2: return array.get(NDArrayIndex.point(idx), NDArrayIndex.all()); case 3: return array.get(NDArrayIndex.interval(idx, idx, true), NDArrayIndex.all(), NDArrayIndex.all()); case 4: return array.get(NDArrayIndex.interval(idx, idx, true), NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.all()); default: throw new IllegalStateException("Cannot get subset for rank " + array.rank() + " array"); } }
/** * Given an all index and * the intended indexes, return an * index array containing a combination of all elements * for slicing and overriding particular indexes where necessary * @param allIndex the index containing all elements * @param intendedIndexes the indexes specified by the user * @return the resolved indexes (containing all where nothing is specified, and the intended index * for a particular dimension otherwise) */ public static INDArrayIndex[] resolve(INDArrayIndex[] allIndex, INDArrayIndex... intendedIndexes) { int numNewAxes = numNewAxis(intendedIndexes); INDArrayIndex[] all = new INDArrayIndex[allIndex.length + numNewAxes]; Arrays.fill(all, NDArrayIndex.all()); for (int i = 0; i < allIndex.length; i++) { //collapse single length indexes in to point indexes if (i >= intendedIndexes.length) break; if (intendedIndexes[i] instanceof NDArrayIndex) { NDArrayIndex idx = (NDArrayIndex) intendedIndexes[i]; if (idx.indices.length == 1) intendedIndexes[i] = new PointIndex(idx.indices[0]); } all[i] = intendedIndexes[i]; } return all; }
public INDArray loadSingleSentence(String sentence) { List<String> tokens = this.tokenizeSentence(sentence); int[] featuresShape = new int[]{1, 1, 0, 0}; if(this.sentencesAlongHeight) { featuresShape[2] = Math.min(this.maxSentenceLength, tokens.size()); featuresShape[3] = this.wordVectorSize; } else { featuresShape[2] = this.wordVectorSize; featuresShape[3] = Math.min(this.maxSentenceLength, tokens.size()); } INDArray features = Nd4j.create(featuresShape); int length = this.sentencesAlongHeight?featuresShape[2]:featuresShape[3]; for(int i = 0; i < length; ++i) { INDArray vector = this.getVector(tokens.get(i)); INDArrayIndex[] indices = new INDArrayIndex[]{NDArrayIndex.point(0), NDArrayIndex.point(0), null, null}; if(this.sentencesAlongHeight) { indices[2] = NDArrayIndex.point(i); indices[3] = NDArrayIndex.all(); } else { indices[2] = NDArrayIndex.all(); indices[3] = NDArrayIndex.point(i); } features.put(indices, vector); } return features; }
INDArrayIndex[] indices = new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(i, sy, iLim), NDArrayIndex.interval(j, sx, jLim)}; INDArray colAdd = col.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(i), NDArrayIndex.point(j), NDArrayIndex.all(), NDArrayIndex.all()); get.addi(colAdd); img.put(indices, get); return img.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(ph, ph + h), NDArrayIndex.interval(pw, pw + w));
/** * Used post training to convert a String to a features INDArray that can be passed to the network output method * * @param reviewContents Contents of the review to vectorize * @param maxLength Maximum length (if review is longer than this: truncate to maxLength). Use Integer.MAX_VALUE to not nruncate * @return Features array for the given input String */ public INDArray loadFeaturesFromString(String reviewContents, int maxLength){ List<String> tokens = tokenizerFactory.create(reviewContents).getTokens(); List<String> tokensFiltered = new ArrayList<>(); for(String t : tokens ){ if(wordVectors.hasWord(t)) tokensFiltered.add(t); } int outputLength = Math.max(maxLength,tokensFiltered.size()); INDArray features = Nd4j.create(1, vectorSize, outputLength); for( int j=0; j<tokens.size() && j<maxLength; j++ ){ String token = tokens.get(j); INDArray vector = wordVectors.getWordVectorMatrix(token); features.put(new INDArrayIndex[]{NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(j)}, vector); } return features; } }
private INDArray labelsMinusMu(INDArray labels, INDArray mu) { // Now that we have the mixtures, let's compute the negative // log likelihodd of the label against the long nSamples = labels.size(0); long labelsPerSample = labels.size(1); // This worked, but was actually much // slower than the for loop below. // labels = samples, mixtures, labels // mu = samples, mixtures // INDArray labelMinusMu = labels // .reshape('f', nSamples, labelsPerSample, 1) // .repeat(2, mMixtures) // .permute(0, 2, 1) // .subi(mu); // The above code does the same thing as the loop below, // but it does it with index magix instead of a for loop. // It turned out to be way less efficient than the simple 'for' here. INDArray labelMinusMu = Nd4j.zeros(nSamples, mMixtures, labelsPerSample); for (int k = 0; k < mMixtures; k++) { labelMinusMu.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.point(k), NDArrayIndex.all()}, labels); } labelMinusMu.subi(mu); return labelMinusMu; }
@Override public INDArray putRow(long row, INDArray toPut) { if (isRowVector() && toPut.isVector()) { return assign(toPut); } return put(new INDArrayIndex[] {NDArrayIndex.point(row), NDArrayIndex.all()}, toPut); }
@Override public INDArray putColumn(int column, INDArray toPut) { if (isColumnVector() && toPut.isVector()) { return assign(toPut); } return put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.point(column)}, toPut); }
int len = ArrayUtil.prod(newShape1); INDArray temp1 = Nd4j.linspace(1, len, len).reshape(newShape1); INDArray subset1 = temp1.get(NDArrayIndex.interval(2, shape[0] + 2), NDArrayIndex.all(), NDArrayIndex.all()); list.add(new Pair<>(subset1, baseMsg + ".get(0)")); int len2 = ArrayUtil.prod(newShape2); INDArray temp2 = Nd4j.linspace(1, len2, len2).reshape(newShape2); INDArray subset2 = temp2.get(NDArrayIndex.all(), NDArrayIndex.interval(3, shape[1] + 3), NDArrayIndex.all()); list.add(new Pair<>(subset2, baseMsg + ".get(1)")); int len3 = ArrayUtil.prod(newShape3); INDArray temp3 = Nd4j.linspace(1, len3, len3).reshape(newShape3); INDArray subset3 = temp3.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(4, shape[2] + 4)); list.add(new Pair<>(subset3, baseMsg + ".get(2)"));
public static INDArray mergePerOutputMasks2d(long[] outShape, INDArray[] arrays, INDArray[] masks) { val numExamplesPerArr = new long[arrays.length]; for (int i = 0; i < numExamplesPerArr.length; i++) { numExamplesPerArr[i] = arrays[i].size(0); } INDArray outMask = Nd4j.ones(outShape); //Initialize to 'all present' (1s) int rowsSoFar = 0; for (int i = 0; i < masks.length; i++) { long thisRows = numExamplesPerArr[i]; //Mask itself may be null -> all present, but may include multiple examples if (masks[i] == null) { continue; } outMask.put(new INDArrayIndex[] {NDArrayIndex.interval(rowsSoFar, rowsSoFar + thisRows), NDArrayIndex.all()}, masks[i]); rowsSoFar += thisRows; } return outMask; }
/** * Insert a row in to this array * Will throw an exception if this * ndarray is not a matrix * * @param row the row insert into * @param toPut the row to insert * @return this */ @Override public INDArray putRow(long row, INDArray toPut) { if (isRowVector() && toPut.isVector()) { return assign(toPut); } return put(new INDArrayIndex[] {NDArrayIndex.point(row), NDArrayIndex.all()}, toPut); }
/** * Get the specified column * * @param c */ @Override public INDArray getColumn(long c) { Nd4j.getCompressor().autoDecompress(this); if (isColumnVector() && c == 0) return this; else if (isColumnVector() && c > 0) throw new IllegalArgumentException("Illegal index for row"); else if(isRowVector()) { return Nd4j.scalar(getDouble(c)); } return get(NDArrayIndex.all(), NDArrayIndex.point(c)); }
/** * Returns the slice of this from the specified dimension * * @param slice the dimension to return from * @param dimension the dimension of the slice to return * @return the slice of this matrix from the specified dimension * and dimension */ @Override public INDArray slice(long slice, int dimension) { Nd4j.getCompressor().autoDecompress(this); long slices = size(dimension); if (slice >= slices) throw new IllegalArgumentException("Illegal slice " + slice); if (Shape.rank(javaShapeInformation) == 0) { if (slice == 0) return createScalarForIndex(slice, true); else throw new IllegalArgumentException("Can't slice a 0-d NDArray"); } if (slice < 0) slice += rank(); INDArrayIndex[] indexes = new INDArrayIndex[rank()]; indexes[dimension] = NDArrayIndex.point(slice); for (int i = 0; i < rank(); i++) { if (i != dimension) indexes[i] = NDArrayIndex.all(); } return get(indexes); }
/** * Insert a column in to this array * Will throw an exception if this * ndarray is not a matrix * * @param column the column to insert * @param toPut the array to put * @return this */ @Override public INDArray putColumn(int column, INDArray toPut) { Nd4j.getCompressor().autoDecompress(this); if (isColumnVector() && toPut.isVector()) { return assign(toPut); } return put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.point(column)}, toPut); }
INDArray currentWindowBernoulli = bernoullis.get(NDArrayIndex.all(), NDArrayIndex.interval(currentTimeSliceStart, currentTimeSliceEnd)); INDArray currentMask = labelMask.get(NDArrayIndex.all(), NDArrayIndex.interval(currentTimeSliceStart, currentTimeSliceEnd)); INDArray currentLabel; if (label.size(1) == 2) { currentLabel = label.get(NDArrayIndex.all(), NDArrayIndex.point(minorityLabel), NDArrayIndex.interval(currentTimeSliceStart, currentTimeSliceEnd)); } else { currentLabel = label.get(NDArrayIndex.all(), NDArrayIndex.point(0), NDArrayIndex.interval(currentTimeSliceStart, currentTimeSliceEnd)); if (minorityLabel == 0) {
indexes[0] = NDArrayIndex.point(slice); for (int i = 1; i < rank(); i++) { indexes[i] = NDArrayIndex.all();
/** * Get a copy of a row. * * @param r the row to get */ @Override public INDArray getRow(long r) { if (isRowVector() && r == 0) return this; else if (isRowVector() && r > 0) throw new IllegalArgumentException("Illegal index for row"); INDArray result = get(NDArrayIndex.point(r), NDArrayIndex.all()); // FIXME: this is bad if (!this.isView() && this.ordering() == 'c' && result.elementWiseStride() == 1 && result.ordering() != 'c') { ((BaseNDArray) result).setShapeInformation(Nd4j.getShapeInfoProvider().createShapeInformation(result.shape(), result.stride(), 0, 1, 'c')); } return result; }