private static String gradientAssertMessage(DoubleVertex outputVertex, DoubleVertex inputVertex, boolean isForwardMode) { String descriptor = isForwardMode ? "Forward" : "Reverse"; return String.format("%s derivative of vertex with ID %s with respect to %s should match finite difference", descriptor, outputVertex.getId(), inputVertex.getId()); }
dOfWrtLatentsAccumulated.putWithRespectTo(vertexWithDiff.getId(), dLogProbOfWrtVertexWithDiff); } else {
expected += expectedPartialDerivatives.getWithRespectTo(scalarVertex.getId()).scalar();
private double calculateStepsize(DoubleVertex vertex, double startingValue) { List<DoubleVertex> vertices = Arrays.asList(vertex); KeanuProbabilisticModelWithGradient model = new KeanuProbabilisticModelWithGradient(vertex.getConnectedGraph()); VertexId vertexId = vertex.getId(); vertex.setValue(DoubleTensor.scalar(startingValue)); Map<VariableReference, DoubleTensor> position = Collections.singletonMap(vertexId, vertex.getValue()); Map<? extends VariableReference, DoubleTensor> gradient = model.logProbGradients(); return Stepsize.findStartingStepSize( position, gradient, Collections.singletonList(vertex), model, ProbabilityCalculator.calculateLogProbFor(vertices), random ); }
@Test public void doesCalculateDiffLogProbWithRespectToHyperParamHandCalculated() { DoubleVertex A = new GaussianVertex(new long[]{1, 2}, 0, 1); A.setValue(new double[]{0.25, 0.6}); DoubleVertex B = new GaussianVertex(new long[]{1, 2}, 0, 1); B.setValue(new double[]{0.5, 0.2}); DoubleVertex C = A.times(B); BernoulliVertex D = new BernoulliVertex(C); D.observe(BooleanTensor.create(new boolean[]{true, false})); LogProbGradientCalculator logProbGradientCalculator = new LogProbGradientCalculator(ImmutableList.of(D), ImmutableList.of(A, B)); Map<VertexId, DoubleTensor> dLogPmf = logProbGradientCalculator.getJointLogProbGradientWrtLatents(); DoubleTensor expectedWrtA = DoubleTensor.create(new double[]{(1.0 / 0.125) * 0.5, (-1.0 / 0.88) * 0.2}); DoubleTensor expectedWrtB = DoubleTensor.create(new double[]{(1.0 / 0.125) * 0.25, (-1.0 / 0.88) * 0.6}); assertEquals(expectedWrtA, dLogPmf.get(A.getId())); assertEquals(expectedWrtB, dLogPmf.get(B.getId())); }
@Test public void calculatesBoundsWhenAllAreSpecified() { DoubleVertex A = new UniformVertex(new long[]{2}, -2, 1); DoubleVertex B = new UniformVertex(new long[]{2}, -2, 1); B.observe(2); DoubleVertex D = A.plus(B); OptimizerBounds bounds = new OptimizerBounds(); bounds.addBound(A.getId(), DoubleTensor.create(-2, -1), 1); ApacheMathSimpleBoundsCalculator boundsCalculator = new ApacheMathSimpleBoundsCalculator(Double.POSITIVE_INFINITY, bounds); ImmutableList<DoubleVertex> latentVertices = ImmutableList.of(A); SimpleBounds simpleBounds = boundsCalculator.getBounds(latentVertices, new double[]{0, 0, 0}); assertArrayEquals(new double[]{-2, -1}, simpleBounds.getLower(), 0.0); assertArrayEquals(new double[]{1, 1}, simpleBounds.getUpper(), 0.0); }
@Test public void doesMatchReverseAutoDiffWithManyOps() { long[] shape = new long[]{2, 2}; DoubleVertex A = new GaussianVertex(shape, 0, 1); A.setValue(DoubleTensor.linspace(0.1, 2, 4).reshape(shape)); DoubleVertex B = new GaussianVertex(shape, 0, 1); B.setValue(DoubleTensor.linspace(0.2, 1, 4).reshape(shape)); DoubleVertex D = A.atan2(B).sigmoid().times(B); DoubleVertex C = A.sin().cos().div(D); DoubleVertex E = C.times(D).pow(A).acos(); DoubleVertex G = E.log().tan().asin().atan(); DoubleVertex F = D.plus(B).exp(); SumVertex H = G.plus(F).sum(); GaussianVertex J = new GaussianVertex(H, 1); J.observe(0.5); LogProbGradientCalculator calculator = new LogProbGradientCalculator(ImmutableList.of(J), ImmutableList.of(A, B)); Map<VertexId, DoubleTensor> gradient = calculator.getJointLogProbGradientWrtLatents(); DoubleTensor dJLogProbWrtAValue = gradient.get(A.getId()); DoubleTensor dJLogProbWrtBValue = gradient.get(B.getId()); PartialsOf dHForward = Differentiator.reverseModeAutoDiff(H, A, B); DoubleTensor dHdA = dHForward.withRespectTo(A); DoubleTensor dHdB = dHForward.withRespectTo(B); DoubleTensor dJLogProbWrtH = J.dLogProbAtValue(H).get(H); DoubleTensor expectedDJLogProbWrtAValue = dJLogProbWrtH.times(dHdA); DoubleTensor expectedDJLogProbWrtBValue = dJLogProbWrtH.times(dHdB); assertEquals(expectedDJLogProbWrtAValue, dJLogProbWrtAValue); assertEquals(expectedDJLogProbWrtBValue, dJLogProbWrtBValue); }
private static double runNonGradientOptimizer(DoubleVertex temperature) { //%%SNIPPET_START%% NonGradientOptimizerMostProbable OptimizerBounds temperatureBounds = new OptimizerBounds().addBound(temperature.getId(), -250., 250.0); NonGradientOptimizer optimizer = KeanuOptimizer.NonGradient.builderFor(temperature.getConnectedGraph()) .maxEvaluations(5000) .boundsRange(100000) .optimizerBounds(temperatureBounds) .initialTrustRegionRadius(5.) .stoppingTrustRegionRadius(2e-8) .build(); optimizer.maxAPosteriori(); double calculatedTemperature = temperature.getValue().scalar(); //%%SNIPPET_END%% NonGradientOptimizerMostProbable return calculatedTemperature; } }
@Test public void canFindMAPGivenBounds() { DoubleVertex A = new GaussianVertex(new long[]{2}, ConstantVertex.of(new double[]{1, -3}), 1); A.setValue(new double[]{0, 0}); OptimizerBounds bounds = new OptimizerBounds(); bounds.addBound(A.getId(), DoubleTensor.create(-1, -2), 0.9); NonGradientOptimizer optimizer = KeanuOptimizer.NonGradient .builderFor(A.getConnectedGraph()) .boundsRange(10) .optimizerBounds(bounds) .build(); optimizer.maxAPosteriori(); assertArrayEquals(new double[]{0.9, -2}, A.getValue().asFlatDoubleArray(), 1e-2); }
@Category(Slow.class) @Test public void youCanChooseSamplingInsteadOfGradientOptimization() { final int smallRawDataSize = 20; final int samplingCount = 5000; LinearRegressionTestUtils.TestData data = LinearRegressionTestUtils.generateSingleFeatureData(smallRawDataSize); ProposalDistribution proposalDistribution = new GaussianProposalDistribution(DoubleTensor.scalar(0.25)); SamplingModelFitting sampling = new SamplingModelFitting(model -> MetropolisHastings.builder() .proposalDistribution(proposalDistribution) .variableSelector(MHStepVariableSelector.SINGLE_VARIABLE_SELECTOR) .rejectionStrategy(new RollbackAndCascadeOnRejection(model.getLatentVertices())) .build(), samplingCount); RegressionModel linearRegressionModel = RegressionModel.withTrainingData(data.xTrain, data.yTrain) .withRegularization(RegressionRegularization.LASSO) .withPriorOnIntercept(0, data.intercept) .withPriorOnWeights( DoubleTensor.create(0., data.weights.getShape()), data.weights ) .withSampling(sampling) .build(); NetworkSamples networkSamples = sampling.getNetworkSamples().drop(samplingCount / 10).downSample(2); assertSampledWeightsAndInterceptMatchTestData( networkSamples.getDoubleTensorSamples(linearRegressionModel.getWeightVertex().getId()), networkSamples.getDoubleTensorSamples(linearRegressionModel.getInterceptVertex().getId()), data); }