.weightInit(WeightInit.DISTRIBUTION) .dist(new NormalDistribution(0,0.2 * (2.0/(4096 + numClasses)))) //This weight init dist gave better results than Xavier .activation(Activation.SOFTMAX).build(), "fc2") .build();
.layer(1, new DenseLayer.Builder().nIn(500).nOut(100).build()) .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) .activation(Activation.SOFTMAX).nIn(100).nOut(10).build()) .pretrain(false).backprop(true) .build();
.layer(1, new DenseLayer.Builder().nIn(500).nOut(100).build()) .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) .activation(Activation.SOFTMAX).nIn(100).nOut(10).build()) .pretrain(false).backprop(true) .build();
@Override public ComputationGraph createComputationalGraph(DomainDescriptor domainDescriptor) { LearningRatePolicy learningRatePolicy = LearningRatePolicy.Poly; layerAssembler.setLearningRatePolicy(learningRatePolicy); layerAssembler.initializeBuilder(); int numInputs = domainDescriptor.getNumInputs("input")[0]; int numHiddenNodes = domainDescriptor.getNumHiddenNodes("firstDense"); ComputationGraphConfiguration.GraphBuilder build = layerAssembler.assemble(numInputs, numHiddenNodes, numLayers); int numIn = layerAssembler.getNumOutputs(); WeightInit WEIGHT_INIT = WeightInit.XAVIER; String lastDenseLayerName = layerAssembler.lastLayerName(); build.addLayer("softmaxGenotype", new OutputLayer.Builder( domainDescriptor.getOutputLoss("softmaxGenotype")) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn) .nOut(domainDescriptor.getNumOutputs("softmaxGenotype")[0]).build(), lastDenseLayerName).addInputs(); appendMetaDataLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); appendIsVariantLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); ComputationGraphConfiguration conf = build .setOutputs(outputNames) .build(); return new ComputationGraph(conf); }
public MultiLayerConfiguration conf() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().trainingWorkspaceMode(workspaceMode) .inferenceWorkspaceMode(workspaceMode).seed(seed).iterations(iterations) .activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new AdaDelta()) .regularization(false).convolutionMode(ConvolutionMode.Same).list() // block 1 .layer(0, new ConvolutionLayer.Builder(new int[] {5, 5}, new int[] {1, 1}).name("cnn1") .nIn(inputShape[0]).nOut(20).activation(Activation.RELU).build()) .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}, new int[] {2, 2}).name("maxpool1").build()) // block 2 .layer(2, new ConvolutionLayer.Builder(new int[] {5, 5}, new int[] {1, 1}).name("cnn2").nOut(50) .activation(Activation.RELU).build()) .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}, new int[] {2, 2}).name("maxpool2").build()) // fully connected .layer(4, new DenseLayer.Builder().name("ffn1").activation(Activation.RELU).nOut(500).build()) // output .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).name("output") .nOut(numLabels).activation(Activation.SOFTMAX) // radial basis function required .build()) .setInputType(InputType.convolutionalFlat(inputShape[2], inputShape[1], inputShape[0])) .backprop(true).pretrain(false).build(); return conf; }
public static void main(String[] args){ //Generate the training data DataSetIterator iterator = getTrainingData(batchSize,rng); //Create the network int numInput = 2; int numOutputs = 1; MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder() .seed(seed) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .weightInit(WeightInit.XAVIER) .updater(new Sgd(learningRate)) .list() .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MSE) .activation(Activation.IDENTITY) .nIn(numInput).nOut(numOutputs).build()) .pretrain(false).backprop(true).build() ); net.init(); net.setListeners(new ScoreIterationListener(1)); for( int i=0; i<nEpochs; i++ ){ iterator.reset(); net.fit(iterator); } final INDArray input = Nd4j.create(new double[] { 0.111111, 0.3333333333333 }, new int[] { 1, 2 }); INDArray out = net.output(input, false); System.out.println(out); }
public static void main(String[] args) throws Exception { //Define a simple ComputationGraph: ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() .weightInit(WeightInit.XAVIER) .updater(new Nesterovs(0.01, 0.9)) .graphBuilder() .addInputs("in") .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.TANH).build(), "in") .addLayer("layer1", new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer0") .setOutputs("layer1") .backprop(true).pretrain(false).build(); ComputationGraph net = new ComputationGraph(conf); net.init(); //Save the model File locationToSave = new File("model/MyComputationGraph.zip"); //Where to save the network. Note: the file is in .zip format - can be opened externally boolean saveUpdater = true; //Updater: i.e., the state for Momentum, RMSProp, Adagrad etc. Save this if you want to train your network more in the future ModelSerializer.writeModel(net, locationToSave, saveUpdater); //Load the model ComputationGraph restored = ModelSerializer.restoreComputationGraph(locationToSave); System.out.println("Saved and loaded parameters are equal: " + net.params().equals(restored.params())); System.out.println("Saved and loaded configurations are equal: " + net.getConfiguration().equals(restored.getConfiguration())); }
/** Returns the network configuration, 2 hidden DenseLayers of size 50. */ private static MultiLayerConfiguration getDeepDenseLayerNetworkConfiguration() { final int numHiddenNodes = 100; return new NeuralNetConfiguration.Builder() .seed(seed) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .weightInit(WeightInit.XAVIER) .updater(new Nesterovs(learningRate, 0.9)) .list() .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes) .activation(Activation.RELU).build()) .layer(1, new DenseLayer.Builder().nIn(numHiddenNodes).nOut(numHiddenNodes) .activation(Activation.RELU).build()) .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MSE) .activation(Activation.IDENTITY) .nIn(numHiddenNodes).nOut(numOutputs).build()) .pretrain(false).backprop(true).build(); }
protected void appendIsVariantLayer(DomainDescriptor domainDescriptor, LearningRatePolicy learningRatePolicy, ComputationGraphConfiguration.GraphBuilder build, int numIn, WeightInit WEIGHT_INIT, String lastDenseLayerName) { if (hasIsVariant) { build.addLayer("isVariant", new OutputLayer.Builder( domainDescriptor.getOutputLoss("isVariant")) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn) .nOut( domainDescriptor.getNumOutputs("isVariant")[0] ).build(), lastDenseLayerName); } }
protected void appendMetaDataLayer(DomainDescriptor domainDescriptor, LearningRatePolicy learningRatePolicy, ComputationGraphConfiguration.GraphBuilder build, int numIn, WeightInit WEIGHT_INIT, String lastDenseLayerName) { build.addLayer("metaData", new OutputLayer.Builder( domainDescriptor.getOutputLoss("metaData")) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn) .nOut( domainDescriptor.getNumOutputs("metaData")[0] ).build(), lastDenseLayerName); }
@Override public OutputLayer getValue(double[] values) { OutputLayer.Builder o = new OutputLayer.Builder(); setLayerOptionsBuilder(o, values); return o.build(); }
.nOut(outputNum) .activation(Activation.SOFTMAX) .build()) .setInputType(InputType.convolutionalFlat(28,28,1)) //See note below .backprop(true).pretrain(false).build();
.nOut(outputNum) .activation(Activation.SOFTMAX) .build()) .setInputType(InputType.convolutionalFlat(28,28,1)) //See note below .backprop(true).pretrain(false).build();
.nOut(outputNum) .activation(Activation.SOFTMAX) .build()) .setInputType(InputType.convolutionalFlat(28,28,1)) //See note below .backprop(true).pretrain(false).build();
.nIn(4096).nOut(numClasses) .weightInit(WeightInit.XAVIER) .activation(Activation.SOFTMAX).build(), "fc2") .build();
.nIn(3*cnnLayerFeatureMaps) .nOut(numFactors) .build(), "globalPool") .setOutputs("out") .build();
public static MultiLayerNetwork lenetModel() { /** * Revisde Lenet Model approach developed by ramgo2 achieves slightly above random * Reference: https://gist.github.com/ramgo2/833f12e92359a2da9e5c2fb6333351c5 **/ MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .seed(seed) .l2(0.005) // tried 0.0001, 0.0005 .activation(Activation.RELU) .weightInit(WeightInit.XAVIER) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .updater(new Nesterovs(0.0001,0.9)) .list() .layer(0, new ConvolutionLayer.Builder(new int[]{5, 5}, new int[]{1, 1}, new int[]{0, 0}).name("cnn1") .nIn(channels).nOut(50).biasInit(0).build()) .layer(1, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool1").build()) .layer(2, new ConvolutionLayer.Builder(new int[]{5,5}, new int[]{5, 5}, new int[]{1, 1}).name("cnn2") .nOut(100).biasInit(0).build()) .layer(3, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool2").build()) .layer(4, new DenseLayer.Builder().nOut(500).build()) .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) .nOut(4) .activation(Activation.SOFTMAX) .build()) .backprop(true).pretrain(false) .setInputType(InputType.convolutional(height, width, channels)) .build(); return new MultiLayerNetwork(conf); }
.lossFunction(LossFunctions.LossFunction.SQUARED_LOSS) .activation(Activation.fromString(outputActivation)) .build()) .pretrain(false).backprop(true) .build();
.build()) .pretrain(false).backprop(true) .build();
public static MultiLayerConfiguration lenetModelConf() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .seed(seed) .l2(0.005) .activation(Activation.RELU) .weightInit(WeightInit.XAVIER) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .updater(new Nesterovs(0.0001, 0.9)) .list() .layer(0, new ConvolutionLayer.Builder(new int[]{5, 5}, new int[]{1, 1}, new int[]{0, 0}).name("cnn1") .nIn(channels).nOut(50).biasInit(0).build()) .layer(1, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool1").build()) .layer(2, new ConvolutionLayer.Builder(new int[]{5,5}, new int[]{5, 5}, new int[]{1, 1}).name("cnn2") .nOut(100).biasInit(0).build()) .layer(3, new SubsamplingLayer.Builder(new int[]{2,2}, new int[]{2,2}).name("maxpool2").build()) .layer(4, new DenseLayer.Builder().nOut(500).build()) .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) .nOut(4) .activation(Activation.SOFTMAX) .build()) .backprop(true).pretrain(false) .setInputType(InputType.convolutional(height, width, channels)) .build(); return conf; } public static void saveModel(FileSystem fs, Model model ) throws Exception{