@Override public INDArray computeScoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); BooleanIndexing.replaceWhere(scoreArr, 0.0, Conditions.lessThan(0.0));//max(0,1-y*yhat) return scoreArr.sum(1); }
@Override public INDArray computeScoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); BooleanIndexing.replaceWhere(scoreArr, 0.0, Conditions.lessThan(0.0));//max(0,1-y*yhat) scoreArr.muli(scoreArr); return scoreArr.sum(1); }
public static void main(String[] args){ int nRows = 3; int nCols = 5; long rngSeed = 12345; //Generate random numbers between -1 and +1 INDArray random = Nd4j.rand(nRows, nCols, rngSeed).muli(2).subi(1); System.out.println("Array values:"); System.out.println(random); //For example, we can conditionally replace values less than 0.0 with 0.0: INDArray randomCopy = random.dup(); BooleanIndexing.replaceWhere(randomCopy, 0.0, Conditions.lessThan(0.0)); System.out.println("After conditionally replacing negative values:\n" + randomCopy); //Or conditionally replace NaN values: INDArray hasNaNs = Nd4j.create(new double[]{1.0,1.0,Double.NaN,1.0}); BooleanIndexing.replaceWhere(hasNaNs,0.0, Conditions.isNan()); System.out.println("hasNaNs after replacing NaNs with 0.0:\n" + hasNaNs); //Or we can conditionally copy values from one array to another: randomCopy = random.dup(); INDArray tens = Nd4j.valueArrayOf(nRows, nCols, 10.0); BooleanIndexing.replaceWhere(randomCopy, tens, Conditions.lessThan(0.0)); System.out.println("Conditionally copying values from array 'tens', if original value is less than 0.0\n" + randomCopy); //One simple task is to count the number of values that match the condition MatchCondition op = new MatchCondition(random, Conditions.greaterThan(0.0)); int countGreaterThanZero = Nd4j.getExecutioner().exec(op,Integer.MAX_VALUE).getInt(0); //MAX_VALUE = "along all dimensions" or equivalently "for entire array" System.out.println("Number of values matching condition 'greater than 0': " + countGreaterThanZero); }
@Override public INDArray getActivation(INDArray in, boolean training) { // no support in ELU native to override alpha if (this.alpha != 1.00) { INDArray alphaMultiple = Nd4j.getExecutioner().execAndReturn(new ELU(in.dup())); alphaMultiple.muli(alpha); BooleanIndexing.replaceWhere(in, alphaMultiple, Conditions.lessThan(0)); } else { Nd4j.getExecutioner().execAndReturn(new ELU(in)); } return in; }
@Override public INDArray getActivation(INDArray in, boolean training) { if (training) { try(MemoryWorkspace ws = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) { this.alpha = Nd4j.rand(in.shape(), l, u, Nd4j.getRandom()); } INDArray inTimesAlpha = in.mul(alpha); BooleanIndexing.replaceWhere(in, inTimesAlpha, Conditions.lessThan(0)); } else { this.alpha = null; double a = 0.5 * (l + u); return Nd4j.getExecutioner().execAndReturn(new RectifedLinear(in, a)); } return in; }
private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray output = activationFn.getActivation(preOutput.dup(), true); if(activationFn instanceof ActivationSoftmax && softmaxClipEps > 0.0){ BooleanIndexing.replaceWhere(output, softmaxClipEps, Conditions.lessThan(softmaxClipEps)); BooleanIndexing.replaceWhere(output, 1.0-softmaxClipEps, Conditions.greaterThan(1.0-softmaxClipEps)); } INDArray scoreArr = Transforms.log(output, false).muli(labels); //Weighted loss function if (weights != null) { if (weights.length() != scoreArr.size(1)) { throw new IllegalStateException("Weights vector (length " + weights.length() + ") does not match output.size(1)=" + preOutput.size(1)); } scoreArr.muliRowVector(weights); } if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0)); BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0));
@Override public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException( "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); INDArray bitMaskRowCol = scoreArr.dup(); /* bit mask is 0 if 1-sigma(y*yhat) is neg, bit mask is 1 if 1-sigma(y*yhat) is +ve */ BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0)); BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0)); INDArray dLda = scoreArr.muli(2).muli(labels.neg()); dLda.muli(bitMaskRowCol); if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) { //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j) //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be // error prone - though buy us a tiny bit of performance LossUtil.applyMask(dLda, mask); } INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params if (mask != null) { LossUtil.applyMask(gradients, mask); } return gradients; }
INDArray lessThan5 = CustomOperations.booleanOp(CustomOperations.randInt(new int[]{3, 3}, 10), Conditions.lessThan(5)); print("Less than 5", lessThan5); Conditions.lessThan(3)); INDArray greaterThan5 = CustomOperations.booleanOp(CustomOperations.randInt(new int[]{3, 3}, 10), Conditions.greaterThan(5));
@Override public INDArray computeScoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); BooleanIndexing.replaceWhere(scoreArr, 0.0, Conditions.lessThan(0.0));//max(0,1-y*yhat) return scoreArr.sum(1); }
@Override public INDArray computeScoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); BooleanIndexing.replaceWhere(scoreArr, 0.0, Conditions.lessThan(0.0));//max(0,1-y*yhat) scoreArr.muli(scoreArr); return scoreArr.sum(1); }
private INDArray ensurePositive(INDArray comboFeatures) { BooleanIndexing.applyWhere(comboFeatures, Conditions.lessThan(0.0f), new Value(0.0f)); BooleanIndexing.applyWhere(comboFeatures, Conditions.greaterThan(0.0f), new Value(1.0f)); return comboFeatures; }
@Override public INDArray getActivation(INDArray in, boolean training) { if (training) { this.alpha = Nd4j.rand(in.shape(), l, u, Nd4j.getRandom()); INDArray inTimesAlpha = in.mul(alpha); BooleanIndexing.replaceWhere(in, inTimesAlpha, Conditions.lessThan(0)); } else { this.alpha = null; double a = 0.5 * (l + u); return Nd4j.getExecutioner().execAndReturn(new RectifedLinear(in, a)); } return in; }
@Override public INDArray getActivation(INDArray in, boolean training) { // no support in ELU native to override alpha if (this.alpha != 1.00) { INDArray alphaMultiple = Nd4j.getExecutioner().execAndReturn(new ELU(in.dup())); alphaMultiple.muli(alpha); BooleanIndexing.replaceWhere(in, alphaMultiple, Conditions.lessThan(0)); } else { Nd4j.getExecutioner().execAndReturn(new ELU(in)); } return in; }
private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException("Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray output = activationFn.getActivation(preOutput.dup(), true); if(activationFn instanceof ActivationSoftmax && softmaxClipEps > 0.0){ BooleanIndexing.replaceWhere(output, softmaxClipEps, Conditions.lessThan(softmaxClipEps)); BooleanIndexing.replaceWhere(output, 1.0-softmaxClipEps, Conditions.greaterThan(1.0-softmaxClipEps)); } INDArray scoreArr = Transforms.log(output, false).muli(labels); //Weighted loss function if (weights != null) { if (weights.length() != scoreArr.size(1)) { throw new IllegalStateException("Weights vector (length " + weights.length() + ") does not match output.size(1)=" + preOutput.size(1)); } scoreArr.muliRowVector(weights); } if (mask != null) { LossUtil.applyMask(scoreArr, mask); } return scoreArr; }
BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0)); BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0));
@Override public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) { if (labels.size(1) != preOutput.size(1)) { throw new IllegalArgumentException("Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer" + " number of outputs (nOut = " + preOutput.size(1) + ") "); } INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask); INDArray bitMaskRowCol = scoreArr.dup(); /* bit mask is 0 if 1-sigma(y*yhat) is neg, bit mask is 1 if 1-sigma(y*yhat) is +ve */ BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0)); BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0)); INDArray dLda = scoreArr.muli(2).muli(labels.neg()); dLda.muli(bitMaskRowCol); if(mask != null && LossUtil.isPerOutputMasking(dLda, mask)){ //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j) //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be // error prone - though buy us a tiny bit of performance LossUtil.applyMask(dLda, mask); } INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params if (mask != null) { LossUtil.applyMask(gradients, mask); } return gradients; }
@Override public void update(INDArray gradient, String paramType) { INDArray yGrads = gradient; gains = gains.add(.2).muli(sign(yGrads)).neqi(sign(yIncs)) .addi(gains.mul(0.8).muli(sign(yGrads)).neqi(sign(yIncs))); BooleanIndexing.applyWhere(gains, Conditions.lessThan(minGain), new Value(minGain)); INDArray gradChange = gains.mul(yGrads); if (useAdaGrad) { if (adaGrad == null) { adaGrad = new AdaGrad(gradient.shape(), learningRate); adaGrad.setStateViewArray(Nd4j.zeros(gradient.shape()).reshape(1, gradChange.length()), gradChange.shape(), gradient.ordering(), true); } gradChange = adaGrad.getGradient(gradChange, 0); } else { gradChange.muli(learningRate); } yIncs.muli(momentum).subi(gradChange); Y.addi(yIncs); }
@Override public void update(INDArray gradient, String paramType) { MemoryWorkspace workspace = workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); try (MemoryWorkspace ws = workspace.notifyScopeEntered()) { INDArray yGrads = gradient; gains = gains.add(.2).muli(sign(yGrads)).neqi(sign(yIncs)) .addi(gains.mul(0.8).muli(sign(yGrads)).neqi(sign(yIncs))); BooleanIndexing.applyWhere(gains, Conditions.lessThan(minGain), new Value(minGain)); INDArray gradChange = gains.mul(yGrads); if (useAdaGrad) { if (adaGrad == null) { // FIXME: int cast adaGrad = new AdaGrad(ArrayUtil.toInts(gradient.shape()), learningRate); adaGrad.setStateViewArray(Nd4j.zeros(gradient.shape()).reshape(1, gradChange.length()), gradChange.shape(), gradient.ordering(), true); } gradChange = adaGrad.getGradient(gradChange, 0); } else { gradChange.muli(learningRate); } yIncs.muli(momentum).subi(gradChange); Y.addi(yIncs); } }