/** * Compute dot product between two vectors. */ public static double dot(SimpleMatrix vector1, SimpleMatrix vector2){ if(vector1.numRows()==1){ // vector1: row vector, assume that vector2 is a row vector too return vector1.mult(vector2.transpose()).get(0); } else if (vector1.numCols()==1){ // vector1: col vector, assume that vector2 is also a column vector. return vector1.transpose().mult(vector2).get(0); } else { throw new AssertionError("Error in neural.Utils.dot: vector1 is a matrix " + vector1.numRows() + " x " + vector1.numCols()); } }
private static SimpleMatrix computeTensorDeltaDown(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector, SimpleMatrix W, SimpleTensor Wt) { SimpleMatrix WTDelta = W.transpose().mult(deltaFull); SimpleMatrix WTDeltaNoBias = WTDelta.extractMatrix(0, deltaFull.numRows() * 2, 0, 1); int size = deltaFull.getNumElements(); SimpleMatrix deltaTensor = new SimpleMatrix(size*2, 1); SimpleMatrix fullVector = NeuralUtils.concatenate(leftVector, rightVector); for (int slice = 0; slice < size; ++slice) { SimpleMatrix scaledFullVector = fullVector.scale(deltaFull.get(slice)); deltaTensor = deltaTensor.plus(Wt.getSlice(slice).plus(Wt.getSlice(slice).transpose()).mult(scaledFullVector)); } return deltaTensor.plus(WTDeltaNoBias); }
/** * @param matrix The matrix to return as a String * @param format The format to use for each value in the matrix, eg "%f" */ public static String toString(SimpleMatrix matrix, String format) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); MatrixIO.print(new PrintStream(stream), matrix.getMatrix(), format); return stream.toString(); }
/** * Applies the derivative of tanh to each of the elements in the vector. Returns a new matrix. */ public static SimpleMatrix elementwiseApplyTanhDerivative(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input.numRows(), input.numCols()); output.set(1.0); output = output.minus(input.elementMult(input)); return output; }
public static SimpleMatrix oneHot(int index, int size) { SimpleMatrix m = new SimpleMatrix(size, 1); m.set(index, 1); return m; }
SimpleMatrix goldLabel = new SimpleMatrix(model.numClasses, 1); int goldClass = RNNCoreAnnotations.getGoldClass(tree); if (goldClass >= 0) { goldLabel.set(goldClass, 1.0); SimpleMatrix deltaClass = goldClass >= 0 ? predictions.minus(goldLabel).scale(nodeWeight) : new SimpleMatrix(predictions.numRows(), predictions.numCols()); SimpleMatrix localCD = deltaClass.mult(NeuralUtils.concatenateWithBias(currentVector).transpose()); double error = -(NeuralUtils.elementwiseApplyLog(predictions).elementMult(goldLabel).elementSum()); error = error * nodeWeight; RNNCoreAnnotations.setPredictionError(tree, error); unaryCD.put(category, unaryCD.get(category).plus(localCD)); SimpleMatrix deltaFromClass = model.getUnaryClassification(category).transpose().mult(deltaClass); deltaFromClass = deltaFromClass.extractMatrix(0, model.op.numHid, 0, 1).elementMult(currentVectorDerivative); SimpleMatrix deltaFull = deltaFromClass.plus(deltaUp); SimpleMatrix oldWordVectorD = wordVectorD.get(word); if (oldWordVectorD == null) { wordVectorD.put(word, deltaFull); } else { wordVectorD.put(word, oldWordVectorD.plus(deltaFull)); String rightCategory = model.basicCategory(tree.children()[1].label().value()); if (model.op.combineClassification) { unaryCD.put("", unaryCD.get("").plus(localCD)); } else { binaryCD.put(leftCategory, rightCategory, binaryCD.get(leftCategory, rightCategory).plus(localCD));
/** * Applies ReLU to each of the entries in the matrix. Returns a new matrix. */ public static SimpleMatrix elementwiseApplyReLU(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input); for (int i = 0; i < output.numRows(); ++i) { for (int j = 0; j < output.numCols(); ++j) { output.set(i, j, Math.max(0, output.get(i, j))); } } return output; }
private static SimpleTensor getTensorGradient(SimpleMatrix deltaFull, SimpleMatrix leftVector, SimpleMatrix rightVector) { int size = deltaFull.getNumElements(); SimpleTensor Wt_df = new SimpleTensor(size*2, size*2, size); // TODO: combine this concatenation with computeTensorDeltaDown? SimpleMatrix fullVector = NeuralUtils.concatenate(leftVector, rightVector); for (int slice = 0; slice < size; ++slice) { Wt_df.setSlice(slice, fullVector.scale(deltaFull.get(slice)).mult(fullVector.transpose())); } return Wt_df; }
/** * Applies softmax to all of the elements of the matrix. The return * matrix will have all of its elements sum to 1. If your matrix is * not already a vector, be sure this is what you actually want. */ public static SimpleMatrix softmax(SimpleMatrix input) { SimpleMatrix output = new SimpleMatrix(input); for (int i = 0; i < output.numRows(); ++i) { for (int j = 0; j < output.numCols(); ++j) { output.set(i, j, Math.exp(output.get(i, j))); } } double sum = output.elementSum(); // will be safe, since exp should never return 0 return output.scale(1.0 / sum); }
public void update(DenseMatrix64F yDense) { // a fast way to make the matrices usable by SimpleMatrix SimpleMatrix y = SimpleMatrix.wrap(yDense); // z = y - H x SimpleMatrix z = y.minus(H.mult(x)); // S = H P H' + R SimpleMatrix S = H.mult(P).mult(H.transpose()).plus(R); // K = PH'S^(-1) SimpleMatrix K = P.mult(H.transpose().mult(S.invert())); // x = x + Kz x = x.plus(K.mult(z)); // P = (I-kH)P = P - KHP P = P.minus(K.mult(H).mult(P)); }
private static SimpleMatrix projectBandwidthToOriginalSpace(SampleModel distribution, double bandwidthFactor) { SimpleMatrix bandwidth = SimpleMatrix.identity(distribution.getGlobalCovariance().numCols()); SimpleMatrix subSpaceBandwidth = distribution.getSubspaceGlobalCovariance().scale(Math.pow(bandwidthFactor, 2)); ArrayList<Integer> subspace = distribution.getmSubspace(); for (int i = 0; i < subSpaceBandwidth.numRows(); i++) { for (int j = 0; j < subSpaceBandwidth.numCols(); j++) { if (subspace.contains(new Integer(i)) && subspace.contains(new Integer(j))) bandwidth.set(i, j, subSpaceBandwidth.get(i, j)); } } SimpleMatrix invSubspaceCov = distribution.getSubspaceInverseCovariance(); bandwidth = invSubspaceCov.transpose().mult(bandwidth).mult(invSubspaceCov); return bandwidth; }
private static double scaleAndRegularize(Map<String, SimpleMatrix> derivatives, Map<String, SimpleMatrix> currentMatrices, double scale, double regCost, boolean activeMatricesOnly, boolean dropBiasColumn) { double cost = 0.0; // the regularization cost for (Map.Entry<String, SimpleMatrix> entry : currentMatrices.entrySet()) { SimpleMatrix D = derivatives.get(entry.getKey()); if (activeMatricesOnly && D == null) { // Fill in an emptpy matrix so the length of theta can match. // TODO: might want to allow for sparse parameter vectors derivatives.put(entry.getKey(), new SimpleMatrix(entry.getValue().numRows(), entry.getValue().numCols())); continue; } SimpleMatrix regMatrix = entry.getValue(); if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.insertIntoThis(0, regMatrix.numCols() - 1, new SimpleMatrix(regMatrix.numRows(), 1)); } D = D.scale(scale).plus(regMatrix.scale(regCost)); derivatives.put(entry.getKey(), D); cost += regMatrix.elementMult(regMatrix).elementSum() * regCost / 2.0; } return cost; }
wordVectorDerivatives.put(word, wordVectorDerivatives.get(word).plus(derivative)); currentVectorDerivative = currentVectorDerivative.elementMult(scoreW.transpose()); SimpleMatrix deltaCurrent = deltaUp.plus(currentVectorDerivative); SimpleMatrix W = dvModel.getWForNode(tree); SimpleMatrix WTdelta = W.transpose().mult(deltaCurrent); binaryScoreDerivatives.get(leftLabel, rightLabel).plus(currentVector.transpose())); SimpleMatrix W_df = deltaCurrent.mult(childrenVector.transpose()); binaryW_dfs.put(leftLabel, rightLabel, binaryW_dfs.get(leftLabel, rightLabel).plus(W_df)); SimpleMatrix leftWTDelta = WTdelta.extractMatrix(0, deltaCurrent.numRows(), 0, 1); SimpleMatrix rightWTDelta = WTdelta.extractMatrix(deltaCurrent.numRows(), deltaCurrent.numRows() * 2, 0, 1); backpropDerivative(tree.children()[0], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, leftDerivative.elementMult(leftWTDelta)); backpropDerivative(tree.children()[1], words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, rightDerivative.elementMult(rightWTDelta)); } else if (tree.children().length == 1) { String childLabel = dvModel.basicCategory(tree.children()[0].label().value()); unaryScoreDerivatives.put(childLabel,unaryScoreDerivatives.get(childLabel).plus(currentVector.transpose())); SimpleMatrix W_df = deltaCurrent.mult(childVectorWithBias.transpose());
protected double computeKineticEnergy(ArrayList<RevoluteJoint> joints, DenseMatrix64F massMatrix) { SimpleMatrix jointVelocities = new SimpleMatrix(joints.size(), 1); for (int i = 0; i < joints.size(); i++) { jointVelocities.set(i, 0, joints.get(i).getQd()); } SimpleMatrix massMatrix_ = SimpleMatrix.wrap(massMatrix); SimpleMatrix kineticEnergy = jointVelocities.transpose().mult(massMatrix_).mult(jointVelocities); return 0.5 * kineticEnergy.get(0, 0); }
/** * @see de.tuhh.luethke.okde.model.BaseSampleDistribution#evaluate(SimpleMatrix pointVector) */ @Override public double evaluate(SimpleMatrix pointVector) { SimpleMatrix smoothedCov = mGlobalCovariance.plus(mBandwidthMatrix); double d = 0d; double n = mGlobalMean.numRows(); double a = Math.pow(Math.sqrt(2 * Math.PI), n); double tmp = (-0.5d) * pointVector.minus(mGlobalMean).transpose().mult(smoothedCov.invert()).mult(pointVector.minus(mGlobalMean)).trace(); d += ((1 / (a * Math.sqrt(smoothedCov.determinant()))) * Math.exp(tmp)) * mGlobalWeight; return d; }
public static double quality( DenseMatrix64F orig , DenseMatrix64F U , DenseMatrix64F W , DenseMatrix64F Vt ) { SimpleMatrix _U = SimpleMatrix.wrap(U); SimpleMatrix _W = SimpleMatrix.wrap(W); SimpleMatrix _Vt = SimpleMatrix.wrap(Vt); SimpleMatrix foundA = _U.mult(_W).mult(_Vt); return SpecializedOps.diffNormF(orig,foundA.getMatrix())/foundA.normF(); }
private static void addTask(DenseMatrix64F taskJacobian, DenseMatrix64F taskObjective, DenseMatrix64F taskWeight, DenseMatrix64F hToModify, DenseMatrix64F fToModify) { SimpleMatrix J = new SimpleMatrix(taskJacobian); SimpleMatrix W = new SimpleMatrix(taskWeight); SimpleMatrix b = new SimpleMatrix(taskObjective); SimpleMatrix H = J.transpose().mult(W).mult(J); SimpleMatrix f = J.transpose().mult(W).mult(b); CommonOps.add(hToModify, H.getMatrix(), hToModify); CommonOps.add(fToModify, f.getMatrix(), fToModify); } }
/** * Copies the data in the slices. Slices are copied rather than * reusing the original SimpleMatrix objects. Each slice must be * the same size. */ public SimpleTensor(SimpleMatrix[] slices) { this.numRows = slices[0].numRows(); this.numCols = slices[0].numCols(); this.numSlices = slices.length; this.slices = new SimpleMatrix[slices.length]; for (int i = 0; i < numSlices; ++i) { if (slices[i].numRows() != numRows || slices[i].numCols() != numCols) { throw new IllegalArgumentException("Slice " + i + " has matrix dimensions " + slices[i].numRows() + "," + slices[i].numCols() + ", expected " + numRows + "," + numCols); } this.slices[i] = new SimpleMatrix(slices[i]); } }
/** * Concatenates several column vectors into one large column * vector, adds a 1.0 at the end as a bias term */ public static SimpleMatrix concatenateWithBias(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } // one extra for the bias size++; SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } result.set(index, 0, 1.0); return result; }
private static double score(SimpleMatrix features, List<SimpleMatrix> weights) { for (int i = 0; i < weights.size(); i += 2) { features = weights.get(i).mult(features).plus(weights.get(i + 1)); if (weights.get(i).numRows() > 1) { features = NeuralUtils.elementwiseApplyReLU(features); } } return features.elementSum(); }