StringBuffer buf = new StringBuffer (); Alphabet dict = getDataAlphabet (); ArrayList thisLabels = new ArrayList (); if (!labelsAtEnd) { while (!isLabelSeparator (toks, j)) { thisLabels.add (labelForTok (toks[j], j)); j++; int firstLblIdx = j; while (j < toks.length) { thisLabels.add (labelForTok (toks[j], j - firstLblIdx)); j++;
public GenericAcrfData2TokenSequence () { super (new Alphabet(), new LabelAlphabet()); labelDicts.add (getTargetAlphabet()); }
basePipe = new GenericAcrfData2TokenSequence (); } else { basePipe = new GenericAcrfData2TokenSequence (numLabelsOption.value); basePipe.setFeaturesIncludeToken(false); basePipe.setIncludeTokenText(false); basePipe.setLabelsAtEnd (labelsAtEnd.value);
public void testNoTokenText () { GenericAcrfData2TokenSequence p = new GenericAcrfData2TokenSequence (2); p.setFeaturesIncludeToken(false); p.setIncludeTokenText(false); InstanceList training = new InstanceList (p); training.addThruPipe (new LineGroupIterator (new StringReader (sampleFixedData), Pattern.compile ("^$"), true)); assertEquals (1, training.size ()); Instance inst1 = training.get (0); LabelsSequence ls1 = (LabelsSequence) inst1.getTarget (); assertEquals (4, ls1.size ()); TokenSequence ts1 = (TokenSequence) inst1.getData (); assertEquals (3, ts1.get(0).getFeatures().size ()); assertEquals (2, ts1.get(1).getFeatures().size ()); }
public void testLabelsAtEnd () throws IOException, ClassNotFoundException { GenericAcrfData2TokenSequence p = new GenericAcrfData2TokenSequence (2); p.setLabelsAtEnd (true); InstanceList training = new InstanceList (p); training.addThruPipe (new LineGroupIterator (new StringReader (labelsAtEndData), Pattern.compile ("^$"), true)); assertEquals (1, training.size ()); Instance inst1 = training.get (0); StringTokenization toks = (StringTokenization) inst1.getData (); LabelsSequence ls1 = (LabelsSequence) inst1.getTarget (); assertEquals (4, ls1.size ()); assertEquals (3, toks.get(0).getFeatures ().size ()); assertEquals ("LBLB LBLD", ls1.getLabels (0).toString ()); LabelAlphabet globalDict = p.getLabelAlphabet (0); assertEquals (2, p.numLevels ()); assertEquals (globalDict, ls1.getLabels (0).get (0).getLabelAlphabet ()); }
public void testFixedNumLabels () throws IOException, ClassNotFoundException { Pipe p = new GenericAcrfData2TokenSequence (2); InstanceList training = new InstanceList (p); training.addThruPipe (new LineGroupIterator (new StringReader (sampleFixedData), Pattern.compile ("^$"), true)); assertEquals (1, training.size ()); Instance inst1 = training.get (0); LabelsSequence ls1 = (LabelsSequence) inst1.getTarget (); assertEquals (4, ls1.size ()); }
basePipe = new GenericAcrfData2TokenSequence (); } else { basePipe = new GenericAcrfData2TokenSequence (numLabelsOption.value); basePipe.setFeaturesIncludeToken(false); basePipe.setIncludeTokenText(false); basePipe.setLabelsAtEnd (labelsAtEnd.value);
public void testNoTokenText () { GenericAcrfData2TokenSequence p = new GenericAcrfData2TokenSequence (2); p.setFeaturesIncludeToken(false); p.setIncludeTokenText(false); InstanceList training = new InstanceList (p); training.addThruPipe (new LineGroupIterator (new StringReader (sampleFixedData), Pattern.compile ("^$"), true)); assertEquals (1, training.size ()); Instance inst1 = training.get (0); LabelsSequence ls1 = (LabelsSequence) inst1.getTarget (); assertEquals (4, ls1.size ()); TokenSequence ts1 = (TokenSequence) inst1.getData (); assertEquals (3, ts1.get(0).getFeatures().size ()); assertEquals (2, ts1.get(1).getFeatures().size ()); }
public void testLabelsAtEnd () throws IOException, ClassNotFoundException { GenericAcrfData2TokenSequence p = new GenericAcrfData2TokenSequence (2); p.setLabelsAtEnd (true); InstanceList training = new InstanceList (p); training.addThruPipe (new LineGroupIterator (new StringReader (labelsAtEndData), Pattern.compile ("^$"), true)); assertEquals (1, training.size ()); Instance inst1 = training.get (0); StringTokenization toks = (StringTokenization) inst1.getData (); LabelsSequence ls1 = (LabelsSequence) inst1.getTarget (); assertEquals (4, ls1.size ()); assertEquals (3, toks.get(0).getFeatures ().size ()); assertEquals ("LBLB LBLD", ls1.getLabels (0).toString ()); LabelAlphabet globalDict = p.getLabelAlphabet (0); assertEquals (2, p.numLevels ()); assertEquals (globalDict, ls1.getLabels (0).get (0).getLabelAlphabet ()); }
public void testFixedNumLabels () throws IOException, ClassNotFoundException { Pipe p = new GenericAcrfData2TokenSequence (2); InstanceList training = new InstanceList (p); training.addThruPipe (new LineGroupIterator (new StringReader (sampleFixedData), Pattern.compile ("^$"), true)); assertEquals (1, training.size ()); Instance inst1 = training.get (0); LabelsSequence ls1 = (LabelsSequence) inst1.getTarget (); assertEquals (4, ls1.size ()); }
basePipe = new GenericAcrfData2TokenSequence (); } else { basePipe = new GenericAcrfData2TokenSequence (numLabelsOption.value); basePipe.setFeaturesIncludeToken(false); basePipe.setIncludeTokenText(false); basePipe.setLabelsAtEnd (labelsAtEnd.value);
basePipe = new GenericAcrfData2TokenSequence (); } else { basePipe = new GenericAcrfData2TokenSequence (numLabelsOption.value); basePipe.setFeaturesIncludeToken(useTokenText.value); basePipe.setIncludeTokenText(useTokenText.value);
StringBuffer buf = new StringBuffer (); Alphabet dict = getDataAlphabet (); ArrayList thisLabels = new ArrayList (); if (!labelsAtEnd) { while (!isLabelSeparator (toks, j)) { thisLabels.add (labelForTok (toks[j], j)); j++; int firstLblIdx = j; while (j < toks.length) { thisLabels.add (labelForTok (toks[j], j - firstLblIdx)); j++;
new GenericAcrfData2TokenSequence (2), new TokenSequence2FeatureVectorSequence (true, true), });
public GenericAcrfData2TokenSequence () { super (new Alphabet(), new LabelAlphabet()); labelDicts.add (getTargetAlphabet()); }
basePipe = new GenericAcrfData2TokenSequence (); } else { basePipe = new GenericAcrfData2TokenSequence (numLabelsOption.value); basePipe.setFeaturesIncludeToken(useTokenText.value); basePipe.setIncludeTokenText(useTokenText.value);
StringBuffer buf = new StringBuffer (); Alphabet dict = getDataAlphabet (); ArrayList thisLabels = new ArrayList (); if (!labelsAtEnd) { while (!isLabelSeparator (toks, j)) { thisLabels.add (labelForTok (toks[j], j)); j++; int firstLblIdx = j; while (j < toks.length) { thisLabels.add (labelForTok (toks[j], j - firstLblIdx)); j++;
new GenericAcrfData2TokenSequence (2), new TokenSequence2FeatureVectorSequence (true, true), });
public GenericAcrfData2TokenSequence () { super (new Alphabet(), new LabelAlphabet()); labelDicts.add (getTargetAlphabet()); }
basePipe = new GenericAcrfData2TokenSequence (); } else { basePipe = new GenericAcrfData2TokenSequence (numLabelsOption.value); basePipe.setFeaturesIncludeToken(useTokenText.value); basePipe.setIncludeTokenText(useTokenText.value);