/** * Concatenates several column vectors into one large column vector */ public static SimpleMatrix concatenate(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } return result; }
/** * Use the given {@code matrix} in place of {@code slice}. * Does not copy the {@code matrix}, but rather uses the actual object. */ public void setSlice(int slice, SimpleMatrix matrix) { if (slice < 0 || slice >= numSlices) { throw new IllegalArgumentException("Unexpected slice number " + slice + " for tensor with " + numSlices + " slices"); } if (matrix.numCols() != numCols) { throw new IllegalArgumentException("Incompatible matrix size. Has " + matrix.numCols() + " columns, tensor has " + numCols); } if (matrix.numRows() != numRows) { throw new IllegalArgumentException("Incompatible matrix size. Has " + matrix.numRows() + " columns, tensor has " + numRows); } slices[slice] = matrix; }
public static List<Double> getPredictionsAsStringList(Tree tree) { SimpleMatrix predictions = getPredictions(tree); List<Double> listOfPredictions = new ArrayList<>(); for (int i = 0 ; i < predictions.numRows() ; i++) { listOfPredictions.add(predictions.get(i)); } return listOfPredictions; }
/** * Init a Map with 0 matrices for all the matrices in the original map. */ private static Map<String, SimpleMatrix> initDerivatives(Map<String, SimpleMatrix> map) { Map<String, SimpleMatrix> derivatives = Generics.newTreeMap(); for (Map.Entry<String, SimpleMatrix> entry : map.entrySet()) { int numRows = entry.getValue().numRows(); int numCols = entry.getValue().numCols(); derivatives.put(entry.getKey(), new SimpleMatrix(numRows, numCols)); } return derivatives; } }
/** * Copies the data in the slices. Slices are copied rather than * reusing the original SimpleMatrix objects. Each slice must be * the same size. */ public SimpleTensor(SimpleMatrix[] slices) { this.numRows = slices[0].numRows(); this.numCols = slices[0].numCols(); this.numSlices = slices.length; this.slices = new SimpleMatrix[slices.length]; for (int i = 0; i < numSlices; ++i) { if (slices[i].numRows() != numRows || slices[i].numCols() != numCols) { throw new IllegalArgumentException("Slice " + i + " has matrix dimensions " + slices[i].numRows() + "," + slices[i].numCols() + ", expected " + numRows + "," + numCols); } this.slices[i] = new SimpleMatrix(slices[i]); } }
private static double score(SimpleMatrix features, List<SimpleMatrix> weights) { for (int i = 0; i < weights.size(); i += 2) { features = weights.get(i).mult(features).plus(weights.get(i + 1)); if (weights.get(i).numRows() > 1) { features = NeuralUtils.elementwiseApplyReLU(features); } } return features.elementSum(); }
/** * Concatenates several column vectors into one large column * vector, adds a 1.0 at the end as a bias term */ public static SimpleMatrix concatenateWithBias(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } // one extra for the bias size++; SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } result.set(index, 0, 1.0); return result; }
/** * Applies log to each of the entries in the matrix. Returns a new matrix. */ public static SimpleMatrix elementwiseApplyLog(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input); for (int i = 0; i < output.numRows(); ++i) { for (int j = 0; j < output.numCols(); ++j) { output.set(i, j, Math.log(output.get(i, j))); } } return output; }
/** * Just for some debug informations in order to compare the results with the scilab computation program. * @param name the name of the matrix * @param m the matrix to print out * @return the String format of the matrix to easily input it to Scilab */ public String toScilabString(String name, SimpleMatrix m) { String result = name + " = ["; for(int i=0;i<m.numRows();++i) { for(int j=0;j<m.numCols();++j) { result += m.get(i, j) + " "; } result += ";"; } return result; }
/** * Applies ReLU to each of the entries in the matrix. Returns a new matrix. */ public static SimpleMatrix elementwiseApplyReLU(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input); for (int i = 0; i < output.numRows(); ++i) { for (int j = 0; j < output.numCols(); ++j) { output.set(i, j, Math.max(0, output.get(i, j))); } } return output; }
/** * Compute dot product between two vectors. */ public static double dot(SimpleMatrix vector1, SimpleMatrix vector2){ if(vector1.numRows()==1){ // vector1: row vector, assume that vector2 is a row vector too return vector1.mult(vector2.transpose()).get(0); } else if (vector1.numCols()==1){ // vector1: col vector, assume that vector2 is also a column vector. return vector1.transpose().mult(vector2).get(0); } else { throw new AssertionError("Error in neural.Utils.dot: vector1 is a matrix " + vector1.numRows() + " x " + vector1.numCols()); } }
private static double scaleAndRegularize(Map<String, SimpleMatrix> derivatives, Map<String, SimpleMatrix> currentMatrices, double scale, double regCost, boolean activeMatricesOnly, boolean dropBiasColumn) { double cost = 0.0; // the regularization cost for (Map.Entry<String, SimpleMatrix> entry : currentMatrices.entrySet()) { SimpleMatrix D = derivatives.get(entry.getKey()); if (activeMatricesOnly && D == null) { // Fill in an emptpy matrix so the length of theta can match. // TODO: might want to allow for sparse parameter vectors derivatives.put(entry.getKey(), new SimpleMatrix(entry.getValue().numRows(), entry.getValue().numCols())); continue; } SimpleMatrix regMatrix = entry.getValue(); if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.insertIntoThis(0, regMatrix.numCols() - 1, new SimpleMatrix(regMatrix.numRows(), 1)); } D = D.scale(scale).plus(regMatrix.scale(regCost)); derivatives.put(entry.getKey(), D); cost += regMatrix.elementMult(regMatrix).elementSum() * regCost / 2.0; } return cost; }
/** * Returns a column vector where each entry is the nth bilinear * product of the nth slices of the two tensors. */ public SimpleMatrix bilinearProducts(SimpleMatrix in) { if (in.numCols() != 1) { throw new AssertionError("Expected a column vector"); } if (in.numRows() != numCols) { throw new AssertionError("Number of rows in the input does not match number of columns in tensor"); } if (numRows != numCols) { throw new AssertionError("Can only perform this operation on a SimpleTensor with square slices"); } SimpleMatrix inT = in.transpose(); SimpleMatrix out = new SimpleMatrix(numSlices, 1); for (int slice = 0; slice < numSlices; ++slice) { double result = inT.mult(slices[slice]).mult(in).get(0); out.set(slice, result); } return out; }
/** * Applies tanh to each of the entries in the matrix. Returns a new matrix. */ public static SimpleMatrix elementwiseApplyTanh(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input); for (int i = 0; i < output.numRows(); ++i) { for (int j = 0; j < output.numCols(); ++j) { output.set(i, j, Math.tanh(output.get(i, j))); } } return output; }
/** * Applies the derivative of tanh to each of the elements in the vector. Returns a new matrix. */ public static SimpleMatrix elementwiseApplyTanhDerivative(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input.numRows(), input.numCols()); output.set(1.0); output = output.minus(input.elementMult(input)); return output; }
/** * Applies softmax to all of the elements of the matrix. The return * matrix will have all of its elements sum to 1. If your matrix is * not already a vector, be sure this is what you actually want. */ public static SimpleMatrix softmax(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input); for (int i = 0; i < output.numRows(); ++i) { for (int j = 0; j < output.numCols(); ++j) { output.set(i, j, Math.exp(output.get(i, j))); } } double sum = output.elementSum(); // will be safe, since exp should never return 0 return output.scale(1.0 / sum); }
/** * Init a TwoDimensionalMap with 0 matrices for all the matrices in the original map. */ private static TwoDimensionalMap<String, String, SimpleMatrix> initDerivatives(TwoDimensionalMap<String, String, SimpleMatrix> map) { TwoDimensionalMap<String, String, SimpleMatrix> derivatives = TwoDimensionalMap.treeMap(); for (TwoDimensionalMap.Entry<String, String, SimpleMatrix> entry : map) { int numRows = entry.getValue().numRows(); int numCols = entry.getValue().numCols(); derivatives.put(entry.getFirstKey(), entry.getSecondKey(), new SimpleMatrix(numRows, numCols)); } return derivatives; }
@SuppressWarnings("unchecked") public Matrix pseudoinverse(double lambda) { SimpleSVD<SimpleMatrix> simpleSVD = this.svd(); SimpleMatrix U = simpleSVD.getU(); SimpleMatrix S = simpleSVD.getW(); SimpleMatrix V = simpleSVD.getV(); int N = Math.min(this.numRows(),this.numCols()); double maxSingular = 0; for( int i = 0; i < N; ++i ) { if( S.get(i, i) > maxSingular ) { maxSingular = S.get(i, i); } } double tolerance = FastMath.DBL_EPSILON * Math.max(this.numRows(),this.numCols()) * maxSingular; for(int i=0;i<Math.min(S.numRows(), S.numCols());++i) { double a = S.get(i, i); if(a <= tolerance) { a = 0; } else { a = a/(a * a + lambda * lambda); } S.set(i, i, a); } return new Matrix(V.mult(S.transpose()).mult(U.transpose())); }
private static SimpleMatrix computeTensorDeltaDown(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector, SimpleMatrix W, SimpleTensor Wt) { SimpleMatrix WTDelta = W.transpose().mult(deltaFull); SimpleMatrix WTDeltaNoBias = WTDelta.extractMatrix(0, deltaFull.numRows() * 2, 0, 1); int size = deltaFull.getNumElements(); SimpleMatrix deltaTensor = new SimpleMatrix(size*2, 1); SimpleMatrix fullVector = NeuralUtils.concatenate(leftVector, rightVector); for (int slice = 0; slice < size; ++slice) { SimpleMatrix scaledFullVector = fullVector.scale(deltaFull.get(slice)); deltaTensor = deltaTensor.plus(Wt.getSlice(slice).plus(Wt.getSlice(slice).transpose()).mult(scaledFullVector)); } return deltaTensor.plus(WTDeltaNoBias); }
private static double scaleAndRegularize(TwoDimensionalMap<String, String, SimpleMatrix> derivatives, TwoDimensionalMap<String, String, SimpleMatrix> currentMatrices, double scale, double regCost, boolean dropBiasColumn) { double cost = 0.0; // the regularization cost for (TwoDimensionalMap.Entry<String, String, SimpleMatrix> entry : currentMatrices) { SimpleMatrix D = derivatives.get(entry.getFirstKey(), entry.getSecondKey()); SimpleMatrix regMatrix = entry.getValue(); if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.insertIntoThis(0, regMatrix.numCols() - 1, new SimpleMatrix(regMatrix.numRows(), 1)); } D = D.scale(scale).plus(regMatrix.scale(regCost)); derivatives.put(entry.getFirstKey(), entry.getSecondKey(), D); cost += regMatrix.elementMult(regMatrix).elementSum() * regCost / 2.0; } return cost; }