public WikiPageParseWorker(final WikiModel model,WikiArticle page){ this.wikiModel = model; this.page = page; textAnnotationBuilder = new CcgTextAnnotationBuilder(new IllinoisTokenizer()); }
private String[] getTokens(String sentence_) { Pair<String[], IntPair[]> lbjTokens = tokenizer.tokenizeSentence(sentence_); return lbjTokens.getFirst(); }
public static void main (String[] args) throws Exception { String docDirInput = "/shared/shelley/yqsong/eventData/ace2005Modify/data/English/"; String docDirOuput = "/shared/experiments/mssammon/aceNer/testEventAce/"; String docDirOuputWithGlobalCoref = "/shared/shelley/yqsong/eventData/ace2005_output_2015/taCacheWithGlobalCoref/"; File outputDir = new File(docDirOuput); if (outputDir.exists() == false) { outputDir.mkdir(); } outputDir = new File(docDirOuputWithGlobalCoref); if (outputDir.exists() == false) { outputDir.mkdir(); } TokenizerTextAnnotationBuilder taBuilder = new TokenizerTextAnnotationBuilder( new IllinoisTokenizer() ); AceFileProcessor fileProcessor = new AceFileProcessor( taBuilder ); annotateAllDocument(fileProcessor, docDirInput, docDirOuput); }
File[] subFolderList = inputFolder.listFiles(); AceFileProcessor processor = new AceFileProcessor( new TokenizerTextAnnotationBuilder( new IllinoisTokenizer() ) ); Set<String> coarseTypes = new HashSet<>(); Set<String> fineTypes = new HashSet<>();
private void initialize(ResourceManager rm_, Comparator<String, EntailmentResult> comparator) throws IOException { ResourceManager fullRm = new SimConfigurator().getConfig(rm_); double threshold = fullRm.getDouble(SimConfigurator.LLM_ENTAILMENT_THRESHOLD.key); tokenizer = new IllinoisTokenizer(); this.comparator = comparator; filter = new WordListFilter(fullRm); neAligner = new Aligner<String, EntailmentResult>(new NEComparator(), filter); aligner = new Aligner<String, EntailmentResult>(comparator, filter); scorer = new GreedyAlignmentScorer<String>(threshold); }