public void importTokensFromTokensFile() { String vocab = getOptionString("tokenVocab"); if ( vocab!=null ) { TokenVocabParser vparser = new TokenVocabParser(this); Map<String,Integer> tokens = vparser.load(); tool.log("grammar", "tokens=" + tokens); for (String t : tokens.keySet()) { if ( t.charAt(0)=='\'' ) defineStringLiteral(t, tokens.get(t)); else defineTokenName(t, tokens.get(t)); } } }
public void importTokensFromTokensFile() { String vocab = getOptionString("tokenVocab"); if ( vocab!=null ) { TokenVocabParser vparser = new TokenVocabParser(this); Map<String,Integer> tokens = vparser.load(); tool.log("grammar", "tokens=" + tokens); for (String t : tokens.keySet()) { if ( t.charAt(0)=='\'' ) defineStringLiteral(t, tokens.get(t)); else defineTokenName(t, tokens.get(t)); } } }
public void importTokensFromTokensFile() { String vocab = getOptionString("tokenVocab"); if ( vocab!=null ) { TokenVocabParser vparser = new TokenVocabParser(this); Map<String,Integer> tokens = vparser.load(); tool.log("grammar", "tokens=" + tokens); for (String t : tokens.keySet()) { if ( t.charAt(0)=='\'' ) defineStringLiteral(t, tokens.get(t)); else defineTokenName(t, tokens.get(t)); } } }
public void importTokensFromTokensFile() { String vocab = getOptionString("tokenVocab"); if ( vocab!=null ) { TokenVocabParser vparser = new TokenVocabParser(this); Map<String,Integer> tokens = vparser.load(); tool.log("grammar", "tokens=" + tokens); for (String t : tokens.keySet()) { if ( t.charAt(0)=='\'' ) defineStringLiteral(t, tokens.get(t)); else defineTokenName(t, tokens.get(t)); } } }
public void importTokensFromTokensFile() { String vocab = getOptionString("tokenVocab"); if ( vocab!=null ) { TokenVocabParser vparser = new TokenVocabParser(this); Map<String,Integer> tokens = vparser.load(); tool.log("grammar", "tokens=" + tokens); for (String t : tokens.keySet()) { if ( t.charAt(0)=='\'' ) defineStringLiteral(t, tokens.get(t)); else defineTokenName(t, tokens.get(t)); } } }