public Void call() throws Exception { for (int ii = start; ii < end; ii++) { if (instancesWithConstraints.get(ii)) { SumLattice lattice = lattices.get(ii); FeatureVectorSequence fvs = (FeatureVectorSequence)data.get(ii).getData(); new GELattice(fvs, lattice.getGammas(), lattice.getXis(), crf, reverseTrans, reverseTransIndices, gradient,this.constraints, false); } } return null; } }
public void computeExpectations(ArrayList<SumLattice> lattices) { double[][][] xis; for (int i = 0; i < lattices.size(); i++) { SumLattice lattice = lattices.get(i); xis = lattice.getXis(); int numStates = xis[0].length; FeatureVectorSequence fvs = (FeatureVectorSequence)lattice.getInput(); for (int ip = 0; ip < fvs.size(); ++ip) { for (int si = 0; si < numStates; si++) { this.expectation += Math.exp(xis[ip][si][si]); } } } System.err.println("Self transition expectation: " + (this.expectation/this.numTokens)); } }
public List<TagResult> tag(Alignment alignment, int nBest) { Instance instance = new Instance(alignment, null, null, null); instance = tduc.getInputPipe().instanceFrom(instance); Sequence inSeq = (Sequence) instance.getData(); List<Sequence<Object>> outSeqs = tduc.getMaxLatticeFactory().newMaxLattice(tduc, inSeq).bestOutputSequences(nBest); ArrayList<TagResult> results = Lists.newArrayListWithCapacity(outSeqs.size()); double z = tduc.getSumLatticeFactory().newSumLattice(tduc, inSeq).getTotalWeight(); for (Sequence<Object> outSeq : outSeqs) { double score = tduc.getSumLatticeFactory().newSumLattice(tduc, inSeq, outSeq).getTotalWeight(); results.add(makeTagResult(outSeq, score - z)); } return results; }
assertTrue(lattice.getGammaProbability(0, crf.getState(0)) == 1.0); assertTrue(lattice.getGammaProbability(0, crf.getState(1)) == 0.0); assertTrue(lattice.getGammaProbability(1, crf.getState(0)) == 0.0); assertTrue(lattice.getGammaProbability(1, crf.getState(1)) == 1.0); .getXiProbability(1, crf.getState(1), crf.getState(1)) == 1.0); assertTrue(lattice .getXiProbability(1, crf.getState(1), crf.getState(0)) == 0.0); assertTrue("Lattice weight = " + lattice.getTotalWeight(), lattice .getTotalWeight() == 4.0); for (int time = 0; time < lattice.length() - 1; time++) { double gammasum = lattice .getGammaProbability(time, crf.getState(0)) + lattice.getGammaProbability(time, crf.getState(1)); assertEquals("Gammas at time step " + time + " sum to " + gammasum, 1.0, gammasum, 0.0001); for (int time = 0; time < lattice.length() - 1; time++) { double xissum = lattice.getXiProbability(time, crf.getState(0), crf .getState(0)) + lattice.getXiProbability(time, crf.getState(0), crf .getState(1)) + lattice.getXiProbability(time, crf.getState(1), crf .getState(0)) + lattice.getXiProbability(time, crf.getState(1), crf .getState(1)); assertEquals("Xis at time step " + time + " sum to " + xissum, 1.0,
if (iter == 0) { assertEquals(defaultLattice.getTotalWeight(), scalingLattice.getTotalWeight(), 0.0001); double[][] g1 = defaultLattice.getGammas(), g2 = scalingLattice .getGammas(); for (int i = 0; i < g1.length; i++) { for (int j = 0; j < g1[i].length; j++) { double[][][] x1 = defaultLattice.getXis(), x2 = scalingLattice .getXis(); for (int i = 0; i < x1.length; i++) { for (int j = 0; j < x1[i].length; j++) {
for (int j = 0; j < targetSeq.size(); j++) { LabelVector predicated = lattice.getLabelingAtPosition(j); Label expected = targetSeq.getLabelAtPosition(j); FeatureVector fv = fvs.get(j);
assertTrue(lattice.getGammaProbability(0, crf.getState(0)) == 1.0); assertTrue(lattice.getGammaProbability(0, crf.getState(1)) == 0.0); assertTrue(lattice.getGammaProbability(1, crf.getState(0)) == 0.0); assertTrue(lattice.getGammaProbability(1, crf.getState(1)) == 1.0); .getXiProbability(1, crf.getState(1), crf.getState(1)) == 1.0); assertTrue(lattice .getXiProbability(1, crf.getState(1), crf.getState(0)) == 0.0); assertTrue("Lattice weight = " + lattice.getTotalWeight(), lattice .getTotalWeight() == 4.0); for (int time = 0; time < lattice.length() - 1; time++) { double gammasum = lattice .getGammaProbability(time, crf.getState(0)) + lattice.getGammaProbability(time, crf.getState(1)); assertEquals("Gammas at time step " + time + " sum to " + gammasum, 1.0, gammasum, 0.0001); for (int time = 0; time < lattice.length() - 1; time++) { double xissum = lattice.getXiProbability(time, crf.getState(0), crf .getState(0)) + lattice.getXiProbability(time, crf.getState(0), crf .getState(1)) + lattice.getXiProbability(time, crf.getState(1), crf .getState(0)) + lattice.getXiProbability(time, crf.getState(1), crf .getState(1)); assertEquals("Xis at time step " + time + " sum to " + xissum, 1.0,
if (iter == 0) { assertEquals(defaultLattice.getTotalWeight(), scalingLattice.getTotalWeight(), 0.0001); double[][] g1 = defaultLattice.getGammas(), g2 = scalingLattice .getGammas(); for (int i = 0; i < g1.length; i++) { for (int j = 0; j < g1[i].length; j++) { double[][][] x1 = defaultLattice.getXis(), x2 = scalingLattice .getXis(); for (int i = 0; i < x1.length; i++) { for (int j = 0; j < x1[i].length; j++) {
LabelVector latticeLabeling = lattice.getLabelingAtPosition(j); double trueLabelProb = latticeLabeling.value(label.getIndex()); int labelIndex = latticeLabeling.getBestIndex();
public Void call() throws Exception { for (int ii = start; ii < end; ii++) { if (instancesWithConstraints.get(ii)) { SumLattice lattice = lattices.get(ii); FeatureVectorSequence fvs = (FeatureVectorSequence)data.get(ii).getData(); new GELattice(fvs, lattice.getGammas(), lattice.getXis(), crf, reverseTrans, reverseTransIndices, gradient,this.constraints, false); } } return null; } }
public void computeExpectations(ArrayList<SumLattice> lattices) { double[][][] xis; for (int i = 0; i < lattices.size(); i++) { SumLattice lattice = lattices.get(i); xis = lattice.getXis(); int numStates = xis[0].length; FeatureVectorSequence fvs = (FeatureVectorSequence)lattice.getInput(); for (int ip = 0; ip < fvs.size(); ++ip) { for (int si = 0; si < numStates; si++) { this.expectation += Math.exp(xis[ip][si][si]); } } } System.err.println("Self transition expectation: " + (this.expectation/this.numTokens)); } }
assert (label != null); LabelVector latticeLabeling = lattice.getLabelingAtPosition(j); double trueLabelProb = latticeLabeling.value(label.getIndex()); int labelIndex = latticeLabeling.getBestIndex();
@Override public List<Alignment> inferAlignments(Word x, int nBest) { Instance instance = new Instance(x.getValue(), null, null, null); instance = crf.getInputPipe().instanceFrom(instance); Sequence inSeq = (Sequence) instance.getData(); List<Sequence<Object>> outSeqs = crf.getMaxLatticeFactory().newMaxLattice(crf, inSeq).bestOutputSequences(nBest); ArrayList<Alignment> results = Lists.newArrayListWithCapacity(outSeqs.size()); double z = crf.getSumLatticeFactory().newSumLattice(crf, inSeq).getTotalWeight(); for (Sequence<Object> outSeq : outSeqs) { double score = crf.getSumLatticeFactory().newSumLattice(crf, inSeq, outSeq).getTotalWeight(); List<Pair<String, String>> graphemes = makeGraphemes(x, outSeq); if (!resultsContain(graphemes, results)) { results.add(new Alignment(x, graphemes, score - z)); } } return results; }
public Void call() throws Exception { for (int ii = start; ii < end; ii++) { if (instancesWithConstraints.get(ii)) { SumLattice lattice = lattices.get(ii); FeatureVectorSequence fvs = (FeatureVectorSequence)data.get(ii).getData(); new GELattice(fvs, lattice.getGammas(), lattice.getXis(), crf, reverseTrans, reverseTransIndices, gradient,this.constraints, false); } } return null; } }
public void computeExpectations(ArrayList<SumLattice> lattices) { double[][][] xis; for (int i = 0; i < lattices.size(); i++) { SumLattice lattice = lattices.get(i); xis = lattice.getXis(); int numStates = xis[0].length; FeatureVectorSequence fvs = (FeatureVectorSequence)lattice.getInput(); for (int ip = 0; ip < fvs.size(); ++ip) { for (int si = 0; si < numStates; si++) { this.expectation += Math.exp(xis[ip][si][si]); } } } System.err.println("Self transition expectation: " + (this.expectation/this.numTokens)); } }
assert (label != null); LabelVector latticeLabeling = lattice.getLabelingAtPosition(j); double trueLabelProb = latticeLabeling.value(label.getIndex()); int labelIndex = latticeLabeling.getBestIndex();