/** * Log on arbitrary base * * @param ndArray * @param base * @return */ public static INDArray log(INDArray ndArray, double base) { return log(ndArray, base, true); }
public static INDArray log(INDArray ndArray) { return log(ndArray, true); }
@Override public void exec(int... dimensions) { if (dimensions[0] != 1) throw new IllegalArgumentException("Only supports row wise calculations"); Nd4j.getExecutioner().exec(new OldSoftMax(x, z)); Transforms.log(z, false); }
public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } /* mean of (yhat - y * log(yhat)) */ //INDArray postOutput = Nd4j.utioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup())); INDArray postOutput = activationFn.getActivation(preOutput.dup(), true); INDArray scoreArr = Transforms.log(postOutput); scoreArr.muli(labels); scoreArr = postOutput.sub(scoreArr); if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
/** * This method returns an array consisting of each of the training samples, * for each label in each sample, the negative log likelihood of that * value falling within the given gaussian mixtures. * @param alpha * @param mu * @param sigma * @param labels * @return */ private INDArray negativeLogLikelihood(INDArray labels, INDArray alpha, INDArray mu, INDArray sigma) { INDArray labelsMinusMu = labelsMinusMu(labels, mu); INDArray diffsquared = labelsMinusMu.mul(labelsMinusMu).sum(2); INDArray phitimesalphasum = phi(diffsquared, sigma).muli(alpha).sum(1); // result = See Bishop(28,29) INDArray result = Transforms.log(phitimesalphasum).negi(); return result; }
private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray output = activationFn.getActivation(preOutput.dup(), true); // Clip output and labels to be between Nd4j.EPS_THREsHOLD and 1, i.e. a valid non-zero probability output = Transforms.min(Transforms.max(output, Nd4j.EPS_THRESHOLD, false), 1, false); labels = Transforms.min(Transforms.max(labels, Nd4j.EPS_THRESHOLD, true), 1, false); INDArray logRatio = Transforms.log(output.rdivi(labels), false); INDArray scoreArr = logRatio.muli(labels); if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray output = activationFn.getActivation(preOutput.dup(), true); if(activationFn instanceof ActivationSoftmax && softmaxClipEps > 0.0){ BooleanIndexing.replaceWhere(output, softmaxClipEps, Conditions.lessThan(softmaxClipEps)); BooleanIndexing.replaceWhere(output, 1.0-softmaxClipEps, Conditions.greaterThan(1.0-softmaxClipEps)); } INDArray scoreArr = Transforms.log(output, false).muli(labels); //Weighted loss function if (weights != null) { if (weights.length() != scoreArr.size(1)) { throw new IllegalStateException("Weights vector (length " + weights.length() + ") does not match output.size(1)=" + preOutput.size(1)); } scoreArr.muliRowVector(weights); } if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray scoreArr; //INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup())); INDArray output = activationFn.getActivation(preOutput.dup(), true); scoreArr = Transforms.log(output.addi(1.0).divi(labels.add(1.0)), false); scoreArr = scoreArr.muli(scoreArr).divi(labels.size(1)); //Weighted loss function if (weights != null) { if (weights.length() != output.size(1)) { throw new IllegalStateException("Weights vector (length " + weights.length() + ") does not match output.size(1)=" + output.size(1)); } scoreArr.muliRowVector(weights); } if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
@Override public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } //INDArray output = Nd4j.getExecutioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup())); INDArray output = activationFn.getActivation(preOutput.dup(), true); INDArray p1 = output.add(1.0); INDArray dlda = p1.rdiv(2.0 / labels.size(1)); INDArray logRatio = Transforms.log(p1.divi(labels.add(1.0)), false); dlda.muli(logRatio); if (weights != null) { dlda.muliRowVector(weights); } if (mask != null && LossUtil.isPerOutputMasking(dlda, mask)) { //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j) //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be // error prone - though buy us a tiny bit of performance LossUtil.applyMask(dlda, mask); } //dL/dz INDArray gradients = activationFn.backprop(preOutput, dlda).getFirst(); //TODO activation functions with weights if (mask != null) { LossUtil.applyMask(gradients, mask); } return gradients; }
print("Vector sin", sin); INDArray log = log(fourByFiveRandomZeroToOne); print("Vector log", log);
@Override public DoubleTensor logInPlace() { Transforms.log(tensor, false); return this; }
/** * Log on arbitrary base * * @param ndArray * @param base * @return */ public static INDArray log(INDArray ndArray, double base) { return log(ndArray, base, Nd4j.copyOnOps); }
public static INDArray log(INDArray ndArray) { return log(ndArray, Nd4j.copyOnOps); }
private INDArray calcLogProbArray(INDArray x, INDArray preOutDistributionParams) { INDArray output = preOutDistributionParams.dup(); activationFn.getActivation(output, false); INDArray logOutput = Transforms.log(output, true); INDArray log1SubOut = Transforms.log(output.rsubi(1.0), false); //For numerical stability: if output = 0, then log(output) == -infinity //then x * log(output) = NaN, but lim(x->0, output->0)[ x * log(output) ] == 0 // therefore: want 0*log(0) = 0, NOT 0*log(0) = NaN by default BooleanIndexing.replaceWhere(logOutput, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0 BooleanIndexing.replaceWhere(log1SubOut, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0 return logOutput.muli(x).addi(x.rsub(1.0).muli(log1SubOut)); }
@Override public INDArray generateRandom(INDArray preOutDistributionParams) { INDArray gamma = activationFn.getActivation(preOutDistributionParams.dup(), false); INDArray lambda = Transforms.exp(gamma, true); //Inverse cumulative distribution function: -log(1-p)/lambda INDArray u = Nd4j.rand(preOutDistributionParams.shape()); //Note here: if u ~ U(0,1) then 1-u ~ U(0,1) return Transforms.log(u, false).divi(lambda).negi(); }
public INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException("Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } /* mean of (yhat - y * log(yhat)) */ //INDArray postOutput = Nd4j.utioner().execAndReturn(Nd4j.getOpFactory().createTransform(activationFn, preOutput.dup())); INDArray postOutput = activationFn.getActivation(preOutput.dup(), true); INDArray scoreArr = Transforms.log(postOutput); scoreArr.muli(labels); scoreArr = postOutput.sub(scoreArr); if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
/** * This method returns an array consisting of each of the training samples, * for each label in each sample, the negative log likelihood of that * value falling within the given gaussian mixtures. * @param alpha * @param mu * @param sigma * @param labels * @return */ private INDArray negativeLogLikelihood(INDArray labels, INDArray alpha, INDArray mu, INDArray sigma) { INDArray labelsMinusMu = labelsMinusMu(labels, mu); INDArray diffsquared = labelsMinusMu.mul(labelsMinusMu).sum(2); INDArray phitimesalphasum = phi(diffsquared, sigma).muli(alpha).sum(1); // result = See Bishop(28,29) INDArray result = Transforms.log( phitimesalphasum ).negi(); return result; }
private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException("Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray output = activationFn.getActivation(preOutput.dup(), true); // Clip output and labels to be between Nd4j.EPS_THREsHOLD and 1, i.e. a valid non-zero probability output = Transforms.min(Transforms.max(output, Nd4j.EPS_THRESHOLD, false), 1, false); labels = Transforms.min(Transforms.max(labels, Nd4j.EPS_THRESHOLD, true), 1, false); INDArray logRatio = Transforms.log(output.rdivi(labels), false); INDArray scoreArr = logRatio.muli(labels); if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
private static INDArray logZ(INDArray z) { INDArray log = log(z, true); // log approaches -Infinity as z approaches zero. Replace -Infinity with the least possible value. // Caveat: does not handle +Infinity since z is assumed to be 0 <= z <= 1. switch (log.data().dataType()) { case FLOAT: BooleanIndexing.applyWhere(log, new Or(Conditions.isNan(), Conditions.isInfinite()), new StableNumber(StableNumber.Type.FLOAT)); break; case DOUBLE: BooleanIndexing.applyWhere(log, new Or(Conditions.isNan(), Conditions.isInfinite()), new StableNumber(StableNumber.Type.DOUBLE)); break; case INT: BooleanIndexing.applyWhere(log, new Or(Conditions.isNan(), Conditions.isInfinite()), new Value(-Integer.MAX_VALUE)); break; default: throw new RuntimeException("unsupported data type: " + log.data().dataType()); } return log; }