@Override public void applyUpdater(INDArray gradient, int iteration, int epoch) { double lr = config.getLearningRate(iteration, epoch); gradient.muli(lr); } }
@Override public void exec() { INDArray softmaxed = Nd4j.getExecutioner().execAndReturn(new OldSoftMax(x)); INDArray mulled = softmaxed.muli(y); INDArray summed = mulled.sum(-1); softmaxed.muliColumnVector(summed); mulled.subi(softmaxed); }
@Override public INDArray computeScoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); return scoreArr.sum(1).muli(-1); }
public void preProcess(INDArray features) { features.divi(this.maxPixelVal); //Scaled to 0->1 if (this.maxRange - this.minRange != 1) features.muli(this.maxRange - this.minRange); //Scaled to minRange -> maxRange if (this.minRange != 0) features.addi(this.minRange); //Offset by minRange }
@Override public void revertFeatures(INDArray features) { if (minRange != 0) { features.subi(minRange); } if (maxRange - minRange != 1.0) { features.divi(maxRange - minRange); } features.muli(this.maxPixelVal); }
@Override public void applyUpdater(INDArray gradient, int iteration, int epoch) { if (lastGradient == null) throw new IllegalStateException("Updater has not been initialized with view state"); double learningRate = config.getLearningRate(iteration, epoch); double rmsDecay = config.getRmsDecay(); double epsilon = config.getEpsilon(); lastGradient.muli(rmsDecay).addi(gradient.mul(gradient).muli(1 - rmsDecay)); // lr * gradient / (sqrt(cache) + 1e-8) gradient.muli(learningRate).divi(Transforms.sqrt(lastGradient.dup(gradientReshapeOrder), false).addi(epsilon)); } }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new SoftSignDerivative(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public void exec() { Nd4j.getExecutioner().exec(new org.nd4j.linalg.api.ops.impl.transforms.TanhDerivative(x, z)); z.muli(wrt()); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new SwishDerivative(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new SELUDerivative(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new Step(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new Step(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new SigmoidDerivative(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new Sigmoid(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public void exec() { Nd4j.getExecutioner().exec(new org.nd4j.linalg.api.ops.impl.transforms.SigmoidDerivative(x,z)); z.muli(wrt()); }
@Override public INDArray computeScoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); BooleanIndexing.replaceWhere(scoreArr, 0.0, Conditions.lessThan(0.0));//max(0,1-y*yhat) scoreArr.muli(scoreArr); return scoreArr.sum(1); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new HardTanhDerivative(in)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) { INDArray dLdz = Nd4j.ones(in.shape()); BooleanIndexing.replaceWhere(dLdz, alpha, Conditions.lessThanOrEqual(0.0)); dLdz.muli(epsilon); return new Pair<>(dLdz, null); }
@Override public INDArray getActivation(INDArray in, boolean training) { // no support in ELU native to override alpha if (this.alpha != 1.00) { INDArray alphaMultiple = Nd4j.getExecutioner().execAndReturn(new ELU(in.dup())); alphaMultiple.muli(alpha); BooleanIndexing.replaceWhere(in, alphaMultiple, Conditions.lessThan(0)); } else { Nd4j.getExecutioner().execAndReturn(new ELU(in)); } return in; }