private static Alphabet dictOfSize (int n) { Alphabet dict = new Alphabet (); for (int i = 0; i < n; i++) { dict.lookupIndex ("feature"+i); } return dict; }
public void dump () { dump (System.out); }
public void addFullyConnectedStatesForLabels () { String[] labels = new String[outputAlphabet.size()]; // This is assuming the the entries in the outputAlphabet are Strings! for (int i = 0; i < outputAlphabet.size(); i++) { labels[i] = (String) outputAlphabet.lookupObject(i); } addFullyConnectedStates (labels); }
private void expandDict (Alphabet fd, int size) { fd.startGrowth (); for (int i = 0; i < size; i++) fd.lookupIndex ("feature"+i, true); }
public CRF4 (Alphabet inputAlphabet, Alphabet outputAlphabet) { inputAlphabet.stopGrowth(); logger.info ("CRF input dictionary size = "+inputAlphabet.size()); //xxx outputAlphabet.stopGrowth(); this.inputAlphabet = inputAlphabet; this.outputAlphabet = outputAlphabet; //this.defaultFeatureIndex = inputAlphabet.size(); }
public void testNotFound () { Alphabet dict = new Alphabet (); dict.lookupIndex ("TEST1"); dict.lookupIndex ("TEST2"); dict.lookupIndex ("TEST3"); assertEquals (-1, dict.lookupIndex ("TEST4", false)); assertEquals (3, dict.size()); assertEquals (3, dict.lookupIndex ("TEST4", true)); }
public AddClassifierTokenPredictions(TokenClassifiers tokenClassifiers, int[] predRanks2add, boolean binary, InstanceList testList) { m_predRanks2add = predRanks2add; m_binary = binary; m_tokenClassifiers = tokenClassifiers; m_inProduction = false; m_dataAlphabet = (Alphabet) tokenClassifiers.getAlphabet().clone(); Alphabet labelAlphabet = tokenClassifiers.getLabelAlphabet(); // add the token prediction features to the alphabet for (int i = 0; i < m_predRanks2add.length; i++) { for (int j = 0; j < labelAlphabet.size(); j++) { String featName = "TOK_PRED=" + labelAlphabet.lookupObject(j).toString() + "_@_RANK_" + m_predRanks2add[i]; m_dataAlphabet.lookupIndex(featName, true); } } // evaluate token classifier if (testList != null) { Trial trial = new Trial(m_tokenClassifiers, testList); logger.info("Token classifier accuracy on test set = " + trial.accuracy()); } }
private void copyStatesAndWeightsFrom (CRF4 initialCRF) { //this.defaultFeatureIndex = initialCRF.defaultFeatureIndex; weightAlphabet = (Alphabet) initialCRF.weightAlphabet.clone (); weights = new SparseVector [initialCRF.weights.length]; states.clear (); for (int i = 0; i < initialCRF.states.size(); i++) { State s = (State) initialCRF.getState (i); String[][] weightNames = new String[s.weightsIndices.length][]; for (int j = 0; j < weightNames.length; j++) { int[] thisW = s.weightsIndices[j]; weightNames[j] = (String[]) initialCRF.weightAlphabet.lookupObjects(thisW, new String [s.weightsIndices[j].length]); } addState (s.name, s.initialCost, s.finalCost, s.destinationNames, s.labels, weightNames); } assert (weights.length > 0); defaultWeights = (double[]) initialCRF.defaultWeights.clone(); for (int i = 0; i < weights.length; i++) { Object wname = weightAlphabet.lookupObject (i); int otherIndex = initialCRF.weightAlphabet.lookupIndex (wname); weights[i] = (SparseVector) initialCRF.weights [otherIndex].cloneMatrix(); } featureSelections = (FeatureSelection[]) initialCRF.featureSelections.clone (); weightsFrozen = (boolean[]) initialCRF.weightsFrozen.clone(); }
public int[] lookupIndices (Object[] objects, boolean addIfNotPresent) { int[] ret = new int[objects.length]; for (int i = 0; i < objects.length; i++) ret[i] = lookupIndex (objects[i], addIfNotPresent); return ret; }
public Object getObjectAtPosition (int pos) { return dictionary.lookupObject (features[pos]); }
public void setAlphabet (Alphabet d) { this.size = d.size(); this.counts = new double[size]; this.dictionary = d; }
private void makeSecondOrderStatesFrom (CRF4 initialCrf) weightAlphabet = new Alphabet (); for (int widx = 0; widx < initialCrf.weightAlphabet.size(); widx++) { getWeightsIndex (initialCrf.getWeightsName (widx)); weightIdxList.add (dest.weightsIndices [j]); int[] widxs = weightIdxList.toNativeArray (); weightNames[j] = (String[]) initialCrf.weightAlphabet.lookupObjects (widxs, new String[widxs.length]); destNames [didx] = s.getName() + LABEL_SEPARATOR + s.destinationNames[didx]; int[] widxs = s.weightsIndices[didx]; weightNames [didx] = (String[]) weightAlphabet.lookupObjects (widxs, new String [widxs.length]);
public TokenSequence2FeatureSequenceWithBigrams (Alphabet dataDict) { super (dataDict, null); biDictionary = new Alphabet(); }
return; Alphabet tmpDV = (Alphabet) ilist.getDataAlphabet().clone(); FeatureSelection featuresSelected = ilist.getFeatureSelection(); InstanceList tmpilist = new InstanceList (tmpDV, ilist.getTargetAlphabet()); ilist.getInstanceWeight(i)); logger.info ("Calculating gradient gain of conjunctions, vocab size = "+tmpDV.size()); RankedFeatureVector gg2 = ranker.newRankedFeatureVector (tmpilist); for (int i = 0; i < 200 && i < gg2.numLocations(); i++) int origVSize = origV.size(); nextfeatures: for (int i = 0; i < gg2.numLocations(); i++) { int index = origV.size()-1;
public void addState (String name, double initialCost, double finalCost, Object[] inputs, Object[] outputs, double[] costs, String[] destinationNames) { this.addState (name, initialCost, finalCost, inputAlphabet.lookupIndices (inputs, true), outputAlphabet.lookupIndices (outputs, true), costs, destinationNames); }
inst.getPipe().getDataAlphabet().setInstanceId (new VMID()); inst.getPipe().getTargetAlphabet().setInstanceId (new VMID());
public void testAddWithPrefix () { Alphabet dict = new Alphabet (); dict.lookupIndex ("ZERO"); dict.lookupIndex ("ONE"); dict.lookupIndex ("TWO"); dict.lookupIndex ("THREE"); FeatureVector fv = new FeatureVector (dict, new int[] { 1,3 }); AugmentableFeatureVector afv = new AugmentableFeatureVector (new Alphabet (), true); afv.add (fv, "O:"); assertEquals (4, dict.size()); assertEquals (2, afv.getAlphabet ().size()); assertEquals ("O:ONE\nO:THREE\n", afv.toString ()); }
public Dirichlet (double[] alphas, Alphabet dict) { if (dict != null && alphas.length != dict.size()) throw new IllegalArgumentException ("alphas and dict sizes do not match."); this.alphas = alphas; this.dict = dict; if (dict != null) dict.stopGrowth(); }
public int lookupIndex (Object entry) { return lookupIndex (entry, true); }
public String getWeightsName (int weightIndex) { return (String) weightAlphabet.lookupObject (weightIndex); }