/** * L2-norm */ public static double l2norm(double[] v) { return Math.sqrt(innerProduct(v, v)); }
/** * Inverse L2-norm */ public static double invL2norm(double[] v) { return 1 / l2norm(v); }
double logSumExp = ArrayMath.logSumOfExps(probs);
alpha.add(val); params[ci] = new Context(ArrayMath.toIntArray(outcomePattern), ArrayMath.toDoubleArray(alpha));
public static double max(double[] x) { int maxIdx = maxIdx(x); return x[maxIdx]; }
/** * Computes \log(\sum_{i=1}^n e^{x_i}) using a maximum-element trick * to avoid arithmetic overflow. * * @param x input vector * @return log-sum of exponentials of vector elements */ public static double logSumOfExps(double[] x) { double max = max(x); double sum = 0.0; for (int i = 0; i < x.length; i++) { if (x[i] != Double.NEGATIVE_INFINITY) sum += Math.exp(x[i] - max); } return max + Math.log(sum); }
l1Cost * ArrayMath.l1norm(nextPoint);
/** * Evaluate the current model on training data set * @return model's training accuracy */ @Override public double evaluate(double[] parameters) { int[][] contexts = indexer.getContexts(); float[][] values = indexer.getValues(); int[] nEventsSeen = indexer.getNumTimesEventsSeen(); int[] outcomeList = indexer.getOutcomeList(); int nOutcomes = indexer.getOutcomeLabels().length; int nPredLabels = indexer.getPredLabels().length; int nCorrect = 0; int nTotalEvents = 0; for (int ei = 0; ei < contexts.length; ei++) { int[] context = contexts[ei]; float[] value = values == null ? null : values[ei]; double[] probs = new double[nOutcomes]; QNModel.eval(context, value, probs, nOutcomes, nPredLabels, parameters); int outcome = ArrayMath.maxIdx(probs); if (outcome == outcomeList[ei]) { nCorrect += nEventsSeen[ei]; } nTotalEvents += nEventsSeen[ei]; } return (double) nCorrect / nTotalEvents; } }
@Override public NegLLComputeTask call() { int ci, oi, ai, vectorIndex, outcome; double predValue, logSumOfExps; negLogLikelihoodThread[threadIndex] = 0; for (ci = startIndex; ci < startIndex + length; ci++) { for (oi = 0; oi < numOutcomes; oi++) { tempSums[oi] = 0; for (ai = 0; ai < contexts[ci].length; ai++) { vectorIndex = indexOf(oi, contexts[ci][ai]); predValue = values != null ? values[ci][ai] : 1.0; tempSums[oi] += predValue * x[vectorIndex]; } } logSumOfExps = ArrayMath.logSumOfExps(tempSums); outcome = outcomeList[ci]; negLogLikelihoodThread[threadIndex] -= (tempSums[outcome] - logSumOfExps) * numTimesEventsSeen[ci]; } return this; } }
/** * L-BFGS two-loop recursion (see Nocedal & Wright 2006, Numerical Optimization, p. 178) */ private void computeDirection(double[] direction) { // Implemented two-loop Hessian update method. int k = updateInfo.kCounter; double[] rho = updateInfo.rho; double[] alpha = updateInfo.alpha; // just to avoid recreating alpha double[][] S = updateInfo.S; double[][] Y = updateInfo.Y; // First loop for (int i = k - 1; i >= 0; i--) { alpha[i] = rho[i] * ArrayMath.innerProduct(S[i], direction); for (int j = 0; j < dimension; j++) { direction[j] = direction[j] - alpha[i] * Y[i][j]; } } // Second loop for (int i = 0; i < k; i++) { double beta = rho[i] * ArrayMath.innerProduct(Y[i], direction); for (int j = 0; j < dimension; j++) { direction[j] = direction[j] + S[i][j] * (alpha[i] - beta); } } for (int i = 0; i < dimension; i++) { direction[i] = -direction[i]; } }
/** * Negative log-likelihood */ public double valueAt(double[] x) { if (x.length != dimension) throw new IllegalArgumentException( "x is invalid, its dimension is not equal to domain dimension."); int ci, oi, ai, vectorIndex, outcome; double predValue, logSumOfExps; double negLogLikelihood = 0; for (ci = 0; ci < numContexts; ci++) { for (oi = 0; oi < numOutcomes; oi++) { tempSums[oi] = 0; for (ai = 0; ai < contexts[ci].length; ai++) { vectorIndex = indexOf(oi, contexts[ci][ai]); predValue = values != null ? values[ci][ai] : 1.0; tempSums[oi] += predValue * x[vectorIndex]; } } logSumOfExps = ArrayMath.logSumOfExps(tempSums); outcome = outcomeList[ci]; negLogLikelihood -= (tempSums[outcome] - logSumOfExps) * numTimesEventsSeen[ci]; } return negLogLikelihood; }
@Override public double valueAt(double[] x) { checkDimension(x); double value = f.valueAt(x); if (l2Cost > 0) { value += l2Cost * ArrayMath.innerProduct(x, x); } return value; }
logSumOfExps = ArrayMath.logSumOfExps(expectation);
double valueAtNextPoint; double dirGradientAtX = ArrayMath.innerProduct(direction, gradAtX);
logSumOfExps = ArrayMath.logSumOfExps(expectation);
double logSumExp = ArrayMath.logSumOfExps(probs); for (int oi = 0; oi < outcomeNames.length; oi++) { probs[oi] = Math.exp(probs[oi] - logSumExp);