@Override public MemoryWorkspace notifyScopeBorrowed(@NonNull T arrayType) { validateConfig(arrayType); enforceExistsAndActive(arrayType); if(scopeOutOfWs.contains(arrayType)){ return Nd4j.getWorkspaceManager().scopeOutOfWorkspaces(); } else { MemoryWorkspace ws = Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( getConfiguration(arrayType), getWorkspaceName(arrayType)); return ws.notifyScopeBorrowed(); } }
@Override public MemoryWorkspace notifyScopeEntered(@NonNull T arrayType) { validateConfig(arrayType); if(isScopedOut(arrayType)){ return Nd4j.getWorkspaceManager().scopeOutOfWorkspaces(); } else { MemoryWorkspace ws = Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( getConfiguration(arrayType), getWorkspaceName(arrayType)); return ws.notifyScopeEntered(); } }
MemoryWorkspace target = Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(id);
try { if (useWorkspace) workspace = Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(configuration, workspaceId);
? new DummyWorkspace() : layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.SINGLE ? Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(workspaceExternal) : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationFeedForward, workspaceFeedForward); Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(workspaceFeedForward).initializeWorkspace();
: Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( ComputationGraph.workspaceConfigurationExternal, ComputationGraph.workspaceExternal); MemoryWorkspace cache = layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( ComputationGraph.workspaceConfigurationCache, ComputationGraph.workspaceCache);
/** Calculate the output of the network, with masking arrays. The masking arrays are used in situations such * as one-to-many and many-to-one recurrent neural network (RNN) designs, as well as for supporting time series * of varying lengths within the same minibatch. */ public INDArray output(INDArray input, boolean train, INDArray featuresMask, INDArray labelsMask) { WorkspaceMode cMode = layerWiseConfigurations.getTrainingWorkspaceMode(); layerWiseConfigurations.setTrainingWorkspaceMode(layerWiseConfigurations.getInferenceWorkspaceMode()); MemoryWorkspace workspace = layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); try (MemoryWorkspace wsE = workspace.notifyScopeEntered()) { INDArray ret = silentOutput(input, train, featuresMask, labelsMask).detach(); layerWiseConfigurations.setTrainingWorkspaceMode(cMode); return ret; } }
/** * Return an array of network outputs (predictions), given the specified network inputs * Network outputs are for output layers only. * * @param train If true: do forward pass at training time; false: do forward pass at test time * @param input Inputs to the network * @return Output activations (order: same as defined in network configuration) */ public INDArray[] output(boolean train, INDArray... input) { WorkspaceMode cMode = configuration.getTrainingWorkspaceMode(); configuration.setTrainingWorkspaceMode(configuration.getInferenceWorkspaceMode()); MemoryWorkspace workspace = configuration.getTrainingWorkspaceMode() == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); try (MemoryWorkspace wsE = workspace.notifyScopeEntered()) { INDArray[] tmp = silentOutput(train, input); for (int x = 0; x < tmp.length; x++) tmp[x] = tmp[x].detach(); configuration.setTrainingWorkspaceMode(cMode); return tmp; } }
@Override public INDArray activate(boolean training) { if (input == null) { throw new IllegalArgumentException("Cannot perform forward pass with null input " + layerId()); } if (cacheMode == null) cacheMode = CacheMode.NONE; applyDropOutIfNecessary(training); INDArray z = preOutput(training); // we do cache only if cache workspace exists. Skip otherwise if (training && cacheMode != CacheMode.NONE && Nd4j.getWorkspaceManager().checkIfWorkspaceExists(ComputationGraph.workspaceCache)) { try (MemoryWorkspace wsB = Nd4j.getWorkspaceManager() .getWorkspaceForCurrentThread(ComputationGraph.workspaceCache).notifyScopeBorrowed()) { preOutput = z.unsafeDuplication(); } } //String afn = conf.getLayer().getActivationFunction(); IActivation afn = layerConf().getActivationFn(); if (helper != null) { INDArray ret = helper.activate(z, layerConf().getActivationFn()); if (ret != null) { return ret; } } INDArray activation = afn.getActivation(z, training); return activation; }
/** * Label the probabilities of the input * * @param input the input to label * @param train whether the output * is test or train. This mainly * affect hyper parameters such as * drop out where certain things should * be applied with activations * @return a vector of probabilities * given each label. * <p> * This is typically of the form: * [0.5, 0.5] or some other probability distribution summing to one */ public INDArray output(INDArray input, boolean train) { WorkspaceMode cMode = layerWiseConfigurations.getTrainingWorkspaceMode(); layerWiseConfigurations.setTrainingWorkspaceMode(layerWiseConfigurations.getInferenceWorkspaceMode()); MemoryWorkspace workspace = layerWiseConfigurations.getTrainingWorkspaceMode() == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); try (MemoryWorkspace wsE = workspace.notifyScopeEntered()) { INDArray ret = silentOutput(input, train).detach(); layerWiseConfigurations.setTrainingWorkspaceMode(cMode); return ret; } }
MemoryWorkspace workspace = configuration.getTrainingWorkspaceMode() == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); try (MemoryWorkspace ws = workspace.notifyScopeEntered()) {
protected void applyDropOutIfNecessary(boolean training) { if (layerConf().getDropOut() > 0 && !conf.isUseDropConnect() && training && !dropoutApplied) { if (Nd4j.getWorkspaceManager().checkIfWorkspaceExists(ComputationGraph.workspaceExternal)) { try (MemoryWorkspace ws = Nd4j.getWorkspaceManager() .getWorkspaceForCurrentThread(ComputationGraph.workspaceExternal) .notifyScopeBorrowed()) { input = input.isView() ? input.dup() : input.unsafeDuplication(); } } else input = input.isView() ? input.dup() : input.unsafeDuplication(); Dropout.applyDropout(input, layerConf().getDropOut()); dropoutApplied = true; } }
: Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal);
: Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal);
@Override public Gradient gradient() { MemoryWorkspace workspace = workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); try (MemoryWorkspace ws = workspace.notifyScopeEntered()) { if (yIncs == null) yIncs = zeros(Y.shape()); if (gains == null) gains = ones(Y.shape()); AtomicDouble sumQ = new AtomicDouble(0); /* Calculate gradient based on barnes hut approximation with positive and negative forces */ INDArray posF = Nd4j.create(Y.shape()); INDArray negF = Nd4j.create(Y.shape()); if (tree == null) { tree = new SpTree(Y); tree.setWorkspaceMode(workspaceMode); } tree.computeEdgeForces(rows, cols, vals, N, posF); for (int n = 0; n < N; n++) tree.computeNonEdgeForces(n, theta, negF.slice(n), sumQ); INDArray dC = posF.subi(negF.divi(sumQ)); Gradient ret = new DefaultGradient(); ret.gradientForVariable().put(Y_GRAD, dC); return ret; } }
: Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal);
: Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); MemoryWorkspace cache = configuration.getTrainingWorkspaceMode() == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(workspaceConfigurationCache, workspaceCache);
: Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal);
MemoryWorkspace target = Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread(id);
@Override public void update(INDArray gradient, String paramType) { MemoryWorkspace workspace = workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( workspaceConfigurationExternal, workspaceExternal); try (MemoryWorkspace ws = workspace.notifyScopeEntered()) { INDArray yGrads = gradient; gains = gains.add(.2).muli(sign(yGrads)).neqi(sign(yIncs)) .addi(gains.mul(0.8).muli(sign(yGrads)).neqi(sign(yIncs))); BooleanIndexing.applyWhere(gains, Conditions.lessThan(minGain), new Value(minGain)); INDArray gradChange = gains.mul(yGrads); if (useAdaGrad) { if (adaGrad == null) { // FIXME: int cast adaGrad = new AdaGrad(ArrayUtil.toInts(gradient.shape()), learningRate); adaGrad.setStateViewArray(Nd4j.zeros(gradient.shape()).reshape(1, gradChange.length()), gradChange.shape(), gradient.ordering(), true); } gradChange = adaGrad.getGradient(gradChange, 0); } else { gradChange.muli(learningRate); } yIncs.muli(momentum).subi(gradChange); Y.addi(yIncs); } }