public static ComputationGraphConfiguration getConf() { ComputationGraphConfiguration.GraphBuilder builder = new NeuralNetConfiguration.Builder() .seed(12345) .updater(new Adam(0.01)) .weightInit(WeightInit.RELU) .graphBuilder() .addInputs("in"); String[] poolNames = new String[ngramFilters.length]; int i = 0; for (int ngram : ngramFilters) { String filterName = String.format("ngram%d", ngram); poolNames[i] = String.format("pool%d", ngram); builder = builder.addLayer(filterName, new Convolution1DLayer.Builder() .nOut(numFilters) .kernelSize(ngram) .activation(Activation.RELU) .build(), "in") .addLayer(poolNames[i], new GlobalPoolingLayer.Builder(PoolingType.MAX).build(), filterName); i++; } return builder.addVertex("concat", new MergeVertex(), poolNames) .addLayer("predict", new DenseLayer.Builder().nOut(numClasses).dropOut(dropoutRetain) .activation(Activation.SOFTMAX).build(), "concat") .addLayer("loss", new LossLayer.Builder(LossFunctions.LossFunction.MCXENT).build(), "predict") .setOutputs("loss") .setInputTypes(InputType.recurrent(W2V_VECTOR_SIZE, 1000)) .build(); } }
.graphBuilder() .addInputs("input") .addLayer("cnn3", new ConvolutionLayer.Builder() .kernelSize(3,vectorSize) .stride(1,vectorSize) .nOut(cnnLayerFeatureMaps) .build(), "input") .addLayer("cnn4", new ConvolutionLayer.Builder() .kernelSize(4,vectorSize) .stride(1,vectorSize) .nOut(cnnLayerFeatureMaps) .build(), "input") .addLayer("cnn5", new ConvolutionLayer.Builder() .kernelSize(5,vectorSize) .stride(1,vectorSize) .build(), "input") .addVertex("merge", new MergeVertex(), "cnn3", "cnn4", "cnn5") //Perform depth concatenation .addLayer("globalPool", new GlobalPoolingLayer.Builder() .poolingType(globalPoolingType) .build(), "merge") .addLayer("out", new OutputLayer.Builder() .lossFunction(LossFunctions.LossFunction.MSE) .activation(Activation.RELU)
/** * Add a layer of the specified configuration to the computation graph * @param layerName * @param layer * @param layerInputs * @return */ public GraphBuilder addLayer(String layerName, Layer layer, String... layerInputs) { initBuilderIfReq(); editedConfigBuilder.addLayer(layerName, layer, null, layerInputs); editedVertices.add(layerName); return this; }
builder.addLayer(lName, l, currentInput); currentInput = lName;
String lastDenseLayerName = layerAssembler.lastLayerName(); build.addLayer("numDistinctAlleles", new OutputLayer.Builder( domainDescriptor.getOutputLoss("numDistinctAlleles")) .weightInit(WEIGHT_INIT) build.addLayer(outputNames[i], new OutputLayer.Builder( domainDescriptor.getOutputLoss(outputNames[i])) .weightInit(WEIGHT_INIT)
@Override public ComputationGraph createComputationalGraph(DomainDescriptor domainDescriptor) { LearningRatePolicy learningRatePolicy = LearningRatePolicy.Poly; layerAssembler.setLearningRatePolicy(learningRatePolicy); layerAssembler.initializeBuilder(); int numInputs = domainDescriptor.getNumInputs("input")[0]; int numHiddenNodes = domainDescriptor.getNumHiddenNodes("firstDense"); ComputationGraphConfiguration.GraphBuilder build = layerAssembler.assemble(numInputs, numHiddenNodes, numLayers); int numIn = layerAssembler.getNumOutputs(); WeightInit WEIGHT_INIT = WeightInit.XAVIER; String lastDenseLayerName = layerAssembler.lastLayerName(); build.addLayer("homozygous", new OutputLayer.Builder( domainDescriptor.getOutputLoss("homozygous")) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn).nOut(11).build(), lastDenseLayerName); for (int i = 1; i < outputNames.length; i++) { build.addLayer(outputNames[i], new OutputLayer.Builder( domainDescriptor.getOutputLoss(outputNames[i])) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn).nOut(2).build(), lastDenseLayerName); } appendMetaDataLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); appendIsVariantLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); ComputationGraphConfiguration conf = build .setOutputs(outputNames) .build(); return new ComputationGraph(conf); }
String shortcutName = "short" + stage + block + "_branch"; graph.addLayer(convName + "2a", new ConvolutionLayer.Builder(new int[] {1, 1}).nOut(filters[0]).cudnnAlgoMode(cudnnAlgoMode) .build(), input) .addLayer(batchName + "2a", new BatchNormalization(), convName + "2a") .addLayer(activationName + "2a", new ActivationLayer.Builder().activation(Activation.RELU).build(), batchName + "2a") .addLayer(convName + "2b", new ConvolutionLayer.Builder(kernelSize).nOut(filters[1]) .cudnnAlgoMode(cudnnAlgoMode).convolutionMode(ConvolutionMode.Same).build(), activationName + "2a") .addLayer(batchName + "2b", new BatchNormalization(), convName + "2b") .addLayer(activationName + "2b", new ActivationLayer.Builder().activation(Activation.RELU).build(), batchName + "2b") .addLayer(convName + "2c", new ConvolutionLayer.Builder(new int[] {1, 1}).nOut(filters[2]) .cudnnAlgoMode(cudnnAlgoMode).build(), activationName + "2b") .addLayer(batchName + "2c", new BatchNormalization(), convName + "2c") .addLayer(convName, new ActivationLayer.Builder().activation(Activation.RELU).build(), shortcutName);
@Override public ComputationGraph createComputationalGraph(DomainDescriptor domainDescriptor) { LearningRatePolicy learningRatePolicy = LearningRatePolicy.Poly; layerAssembler.setLearningRatePolicy(learningRatePolicy); layerAssembler.initializeBuilder(); int numInputs = domainDescriptor.getNumInputs("input")[0]; int numHiddenNodes = domainDescriptor.getNumHiddenNodes("firstDense"); ComputationGraphConfiguration.GraphBuilder build = layerAssembler.assemble(numInputs, numHiddenNodes, numLayers); int numIn = layerAssembler.getNumOutputs(); WeightInit WEIGHT_INIT = WeightInit.XAVIER; String lastDenseLayerName = layerAssembler.lastLayerName(); String combined = fixRef?"combinedRef":"combined"; build.addLayer(combined, new OutputLayer.Builder( domainDescriptor.getOutputLoss(combined)) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn) .nOut(domainDescriptor.getNumOutputs(combined)[0]).build(), lastDenseLayerName); appendMetaDataLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); appendIsVariantLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); ComputationGraphConfiguration conf = build .setOutputs(getOutputNames()) .build(); return new ComputationGraph(conf); }
@Override public GraphConfiguration getValue(double[] values) { //Create ComputationGraphConfiguration... NeuralNetConfiguration.Builder builder = randomGlobalConf(values); ComputationGraphConfiguration.GraphBuilder graphBuilder = builder.graphBuilder(); graphBuilder.addInputs(this.networkInputs); graphBuilder.setOutputs(this.networkOutputs); if (inputTypes != null) graphBuilder.setInputTypes(inputTypes.getValue(values)); //Build/add our layers and vertices: for (LayerConf c : layerSpaces) { org.deeplearning4j.nn.conf.layers.Layer l = c.layerSpace.getValue(values); graphBuilder.addLayer(c.getLayerName(), l, c.getInputs()); } for (VertexConf gv : vertices) { graphBuilder.addVertex(gv.getVertexName(), gv.getGraphVertex(), gv.getInputs()); } if (backprop != null) graphBuilder.backprop(backprop.getValue(values)); if (pretrain != null) graphBuilder.pretrain(pretrain.getValue(values)); if (backpropType != null) graphBuilder.backpropType(backpropType.getValue(values)); if (tbpttFwdLength != null) graphBuilder.tBPTTForwardLength(tbpttFwdLength.getValue(values)); if (tbpttBwdLength != null) graphBuilder.tBPTTBackwardLength(tbpttBwdLength.getValue(values)); ComputationGraphConfiguration configuration = graphBuilder.build(); return new GraphConfiguration(configuration, earlyStoppingConfiguration, numEpochs); }
ComputationGraphConfiguration.GraphBuilder assemble(int numInputs, int numHiddenNodes, int numLayers, String baseLayer, int startingIndex) { assert numHiddenNodes > 0 : "model capacity is too small. At least some hidden nodes must be created."; WeightInit WEIGHT_INIT = WeightInit.XAVIER; float reduction = 1f; int minimum = (int) (numHiddenNodes * Math.pow(reduction, numLayers)); assert minimum > 2 : "Too much reduction, not enough outputs: "; int numIn = numInputs; int numOut = (int) (numHiddenNodes * modelCapacity); String previousLayerName; String lastDenseLayerName = "no layers"; for (int i = startingIndex; i < startingIndex + numLayers; i++) { // System.out.printf("layer %d numIn=%d numOut=%d%n", i, numIn, numOut); lastDenseLayerName = "dense" + i; previousLayerName = i == startingIndex ? baseLayer : "dense" + (i - 1); numOut = (int) (numHiddenNodes * Math.pow(reductionRate, i) * modelCapacity); build.addLayer(lastDenseLayerName, new DenseLayer.Builder().nIn(numIn).nOut(numOut) .weightInit(WEIGHT_INIT) .activation("relu").learningRateDecayPolicy(learningRatePolicy).epsilon(LAYER_EPSILON) .build(), previousLayerName); numIn = numOut; } this.numOutputs = numOut; this.lastLayerName = lastDenseLayerName; return build; }
@Override public ComputationGraph createComputationalGraph(DomainDescriptor domainDescriptor) { LearningRatePolicy learningRatePolicy = LearningRatePolicy.Poly; layerAssembler.setLearningRatePolicy(learningRatePolicy); layerAssembler.initializeBuilder(); int numInputs = domainDescriptor.getNumInputs("input")[0]; int numHiddenNodes = domainDescriptor.getNumHiddenNodes("firstDense"); ComputationGraphConfiguration.GraphBuilder build = layerAssembler.assemble(numInputs, numHiddenNodes, numLayers); int numIn = layerAssembler.getNumOutputs(); WeightInit WEIGHT_INIT = WeightInit.XAVIER; String lastDenseLayerName = layerAssembler.lastLayerName(); build.addLayer("softmaxGenotype", new OutputLayer.Builder( domainDescriptor.getOutputLoss("softmaxGenotype")) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn) .nOut(domainDescriptor.getNumOutputs("softmaxGenotype")[0]).build(), lastDenseLayerName).addInputs(); appendMetaDataLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); appendIsVariantLayer(domainDescriptor, learningRatePolicy, build, numIn, WEIGHT_INIT, lastDenseLayerName); ComputationGraphConfiguration conf = build .setOutputs(outputNames) .build(); return new ComputationGraph(conf); }
@Override public ComputationGraph init() { int embeddingSize = 128; ComputationGraphConfiguration.GraphBuilder graph = graphBuilder("input1"); graph.addInputs("input1").setInputTypes(InputType.convolutional(inputShape[2], inputShape[1], inputShape[0])) // Logits .addLayer("bottleneck", new DenseLayer.Builder().nIn(5376).nOut(embeddingSize).build(), "avgpool") // Embeddings .addVertex("embeddings", new L2NormalizeVertex(new int[] {1}, 1e-10), "bottleneck") // Output .addLayer("outputLayer", new CenterLossOutputLayer.Builder() .lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) .activation(Activation.SOFTMAX).alpha(0.9).lambda(1e-4) .nIn(embeddingSize).nOut(numClasses).build(), "embeddings") .setOutputs("outputLayer").backprop(true).pretrain(false); ComputationGraphConfiguration conf = graph.build(); ComputationGraph model = new ComputationGraph(conf); model.init(); return model; }
public static void main(String[] args) throws Exception { //Define a simple ComputationGraph: ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() .weightInit(WeightInit.XAVIER) .updater(new Nesterovs(0.01, 0.9)) .graphBuilder() .addInputs("in") .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.TANH).build(), "in") .addLayer("layer1", new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer0") .setOutputs("layer1") .backprop(true).pretrain(false).build(); ComputationGraph net = new ComputationGraph(conf); net.init(); //Save the model File locationToSave = new File("model/MyComputationGraph.zip"); //Where to save the network. Note: the file is in .zip format - can be opened externally boolean saveUpdater = true; //Updater: i.e., the state for Momentum, RMSProp, Adagrad etc. Save this if you want to train your network more in the future ModelSerializer.writeModel(net, locationToSave, saveUpdater); //Load the model ComputationGraph restored = ModelSerializer.restoreComputationGraph(locationToSave); System.out.println("Saved and loaded parameters are equal: " + net.params().equals(restored.params())); System.out.println("Saved and loaded configurations are equal: " + net.getConfiguration().equals(restored.getConfiguration())); }
/** * Default layer setup: Create sequential layer network defined by the order of the layer list * * @param gb GraphBuilder object */ protected void makeDefaultLayerSetup(GraphBuilder gb) { String currentInput = "input"; gb.addInputs(currentInput); // Collect layers for (Layer layer : layers) { String lName = layer.getLayerName(); gb.addLayer(lName, layer.getBackend().clone(), currentInput); currentInput = lName; } gb.setOutputs(currentInput); }
protected void appendIsVariantLayer(DomainDescriptor domainDescriptor, LearningRatePolicy learningRatePolicy, ComputationGraphConfiguration.GraphBuilder build, int numIn, WeightInit WEIGHT_INIT, String lastDenseLayerName) { if (hasIsVariant) { build.addLayer("isVariant", new OutputLayer.Builder( domainDescriptor.getOutputLoss("isVariant")) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn) .nOut( domainDescriptor.getNumOutputs("isVariant")[0] ).build(), lastDenseLayerName); } }
protected void appendMetaDataLayer(DomainDescriptor domainDescriptor, LearningRatePolicy learningRatePolicy, ComputationGraphConfiguration.GraphBuilder build, int numIn, WeightInit WEIGHT_INIT, String lastDenseLayerName) { build.addLayer("metaData", new OutputLayer.Builder( domainDescriptor.getOutputLoss("metaData")) .weightInit(WEIGHT_INIT) .activation("softmax").weightInit(WEIGHT_INIT).learningRateDecayPolicy(learningRatePolicy) .nIn(numIn) .nOut( domainDescriptor.getNumOutputs("metaData")[0] ).build(), lastDenseLayerName); }
/** * Add a layer with a specified preprocessor * @param layerName * @param layer * @param preProcessor * @param layerInputs * @return */ public GraphBuilder addLayer(String layerName, Layer layer, InputPreProcessor preProcessor, String... layerInputs) { initBuilderIfReq(); editedConfigBuilder.addLayer(layerName, layer, preProcessor, layerInputs); editedVertices.add(layerName); return this; }
private GraphBuilder inception(GraphBuilder graph, String name, int inputSize, int[][] config, String inputLayer) { graph.addLayer(name + "-cnn1", conv1x1(inputSize, config[0][0], 0.2), inputLayer) .addLayer(name + "-cnn2", c3x3reduce(inputSize, config[1][0], 0.2), inputLayer) .addLayer(name + "-cnn3", c5x5reduce(inputSize, config[2][0], 0.2), inputLayer) .addLayer(name + "-max1", maxPool3x3(1), inputLayer) .addLayer(name + "-cnn4", conv3x3(config[1][0], config[1][1], 0.2), name + "-cnn2") .addLayer(name + "-cnn5", conv5x5(config[2][0], config[2][1], 0.2), name + "-cnn3") .addLayer(name + "-cnn6", conv1x1(inputSize, config[3][0], 0.2), name + "-max1") .addVertex(name + "-depthconcat1", new MergeVertex(), name + "-cnn1", name + "-cnn4", name + "-cnn5", name + "-cnn6"); return graph; }
/** * Add a layer, with no {@link InputPreProcessor}, with the specified name and specified inputs. * * @param layerName Name/label of the layer to add * @param layer The layer configuration * @param layerInputs Inputs to this layer (must be 1 or more). Inputs may be other layers, GraphVertex objects, * on a combination of the two. * @see #addLayer(String, Layer, InputPreProcessor, String...) */ public GraphBuilder addLayer(String layerName, Layer layer, String... layerInputs) { return addLayer(layerName, layer, null, layerInputs); }
DarknetHelper.addLayers(b, 5, 2, 256, 512, 2); //8x8 out b.addLayer("convolution2d_6", new ConvolutionLayer.Builder(1, 1) .nIn(512) .nOut(TinyImageNetFetcher.NUM_LABELS) .activation(Activation.IDENTITY) .build(), "maxpooling2d_5") .addLayer("globalpooling", new GlobalPoolingLayer.Builder(PoolingType.AVG).build(), "convolution2d_6") .addLayer("loss", new LossLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).build(), "globalpooling") .setOutputs("loss");