public void writeModelsToDisk(String dir, String modelName){ IOUtils.mkdir(dir); chunker.write(dir + File.separator + modelName + ".lc", dir + File.separator + modelName + ".lex"); logger.info("Done training, models are in " + dir+File.separator+modelName+".lc (.lex)"); } public static void main(String[] args) {
public void writeModelsToDisk(String dir, String modelName){ IOUtils.mkdir(dir); chunker.write(dir + File.separator + modelName + ".lc", dir + File.separator + modelName + ".lex"); logger.info("Done training, models are in " + dir+File.separator+modelName+".lc (.lex)"); } public static void main(String[] args) {
/** * print output into a file in directory specified, with name based on annotationFile. * Should not create an empty file (i.e., if columnOutput is empty). * * @param nerOutputDir directory to write output file * @param annotationFile used as prefix for the name of the new file * @param columnOutput a list of strings to be printed to the output file * @throws IOException */ private static void printOut(String nerOutputDir, String annotationFile, List<String> columnOutput) throws IOException { String outFile = nerOutputDir + "/" + annotationFile + ".ner.column.txt" ; if ( !columnOutput.isEmpty() ) { if ( !IOUtils.exists( nerOutputDir ) ) IOUtils.mkdir( nerOutputDir ); LineIO.write(outFile, columnOutput); } }
/** * Saves the ".lc" and ".lex" models to disk in the modelPath specified by the constructor The * modelName ("Chunker", as specified in ChunkerConfigurator) is fixed */ public void writeModelsToDisk() { IOUtils.mkdir(rm.getString("modelDirPath")); chunker.save(); logger.info("Done training, models are in " + rm.getString("modelDirPath")); } public void writeModelsToDisk(String dir, String modelName){
/** * Saves the ".lc" and ".lex" models to disk in the modelPath specified by the constructor The * modelName ("Chunker", as specified in ChunkerConfigurator) is fixed */ public void writeModelsToDisk() { IOUtils.mkdir(rm.getString("modelDirPath")); chunker.save(); logger.info("Done training, models are in " + rm.getString("modelDirPath")); } public void writeModelsToDisk(String dir, String modelName){
IOUtils.mkdir(outDir);
String tmpFile = tmpDir + "/google.ngrams.get1t" + (new Random()).nextInt(); IOUtils.mkdir(tmpDir);
/** * This saves an individual TextAnnotation to the desired output folder. * @param foldertype * @param path * @param ta * @throws IOException */ public static void save(String foldertype, String path, TextAnnotation ta) throws IOException { if(!IOUtils.exists(path)) { IOUtils.mkdir(path); } if(foldertype.equals(Common.FOLDERTA)) { SerializationHelper.serializeTextAnnotationToFile(ta, path + "/" + ta.getId(), true); }else if(foldertype.equals(Common.FOLDERTAJSON)) { SerializationHelper.serializeTextAnnotationToFile(ta, path + "/" + ta.getId(), true, true); }else if(foldertype.equals(Common.FOLDERCONLL)) { CoNLLNerReader.TaToConll(Collections.singletonList(ta), path); }else if(foldertype.equals(Common.FOLDERCOLUMN)) { ColumnReader.TaToColumn(Collections.singletonList(ta), path); } }
String tmpFile = tmpDir + "/google.ngrams.get1t" + (new Random()).nextInt(); IOUtils.mkdir(tmpDir);
IOUtils.mkdir(outDir);
IOUtils.mkdir(outDir);
String cacheDBDir = "data-cached"; if (!IOUtils.exists(cacheDBDir)) IOUtils.mkdir(cacheDBDir); String cacheDB = cacheDBDir + File.separator + viewName + "-cache.db"; dbHandler = new TextAnnotationMapDBHandler(cacheDB);
String cacheDBDir = "data-cached"; if (!IOUtils.exists(cacheDBDir)) IOUtils.mkdir(cacheDBDir); String cacheDB = cacheDBDir + File.separator + viewName + "-cache.db"; dbHandler = new TextAnnotationMapDBHandler(cacheDB);
String outDir = args[1]; IOUtils.mkdir(outDir);
System.exit(-1); } else IOUtils.mkdir(conllDir);
System.exit(-1); } else IOUtils.mkdir(conllDir);
public void train() { if (!IOUtils.exists(modelsDir)) IOUtils.mkdir(modelsDir); Learner classifier = new PrepSRLClassifier(modelName + ".lc", modelName + ".lex"); Parser trainDataReader = new PrepSRLDataReader(dataDir, "train"); BatchTrainer trainer = new BatchTrainer(classifier, trainDataReader, 1000); trainer.train(20); classifier.save(); trainDataReader.close(); }
public void train() { if (!IOUtils.exists(modelsDir)) IOUtils.mkdir(modelsDir); Learner classifier = new PrepSRLClassifier(modelName + ".lc", modelName + ".lex"); Parser trainDataReader = new PrepSRLDataReader(dataDir, "train"); BatchTrainer trainer = new BatchTrainer(classifier, trainDataReader, 1000); trainer.train(20); classifier.save(); trainDataReader.close(); }
@CommandDescription( description = "Pre-extracts the features for the verb-sense model. Run this before training.", usage = "preExtract") public static void preExtract() throws Exception { SenseManager manager = getManager(true); ResourceManager conf = new VerbSenseConfigurator().getDefaultConfig(); // If models directory doesn't exist create it if (!IOUtils.isDirectory(conf.getString(conf .getString(VerbSenseConfigurator.MODELS_DIRECTORY)))) IOUtils.mkdir(conf.getString(conf.getString(VerbSenseConfigurator.MODELS_DIRECTORY))); int numConsumers = Runtime.getRuntime().availableProcessors(); Dataset dataset = Dataset.PTBTrainDev; log.info("Pre-extracting features"); ModelInfo modelInfo = manager.getModelInfo(); String featureSet = "" + modelInfo.featureManifest.getIncludedFeatures().hashCode(); String allDataCacheFile = VerbSenseConfigurator.getFeatureCacheFile(featureSet, dataset, rm); FeatureVectorCacheFile featureCache = preExtract(numConsumers, manager, dataset, allDataCacheFile); pruneFeatures(numConsumers, manager, featureCache, VerbSenseConfigurator.getPrunedFeatureCacheFile(featureSet, rm)); Lexicon lexicon = modelInfo.getLexicon().getPrunedLexicon(manager.getPruneSize()); log.info("Saving lexicon with {} features to {}", lexicon.size(), manager.getLexiconFileName()); log.info(lexicon.size() + " features in the lexicon"); lexicon.save(manager.getLexiconFileName()); }
@CommandDescription( description = "Pre-extracts the features for the verb-sense model. Run this before training.", usage = "preExtract") public static void preExtract() throws Exception { SenseManager manager = getManager(true); ResourceManager conf = new VerbSenseConfigurator().getDefaultConfig(); // If models directory doesn't exist create it if (!IOUtils.isDirectory(conf.getString(conf .getString(VerbSenseConfigurator.MODELS_DIRECTORY)))) IOUtils.mkdir(conf.getString(conf.getString(VerbSenseConfigurator.MODELS_DIRECTORY))); int numConsumers = Runtime.getRuntime().availableProcessors(); Dataset dataset = Dataset.PTBTrainDev; log.info("Pre-extracting features"); ModelInfo modelInfo = manager.getModelInfo(); String featureSet = "" + modelInfo.featureManifest.getIncludedFeatures().hashCode(); String allDataCacheFile = VerbSenseConfigurator.getFeatureCacheFile(featureSet, dataset, rm); FeatureVectorCacheFile featureCache = preExtract(numConsumers, manager, dataset, allDataCacheFile); pruneFeatures(numConsumers, manager, featureCache, VerbSenseConfigurator.getPrunedFeatureCacheFile(featureSet, rm)); Lexicon lexicon = modelInfo.getLexicon().getPrunedLexicon(manager.getPruneSize()); log.info("Saving lexicon with {} features to {}", lexicon.size(), manager.getLexiconFileName()); log.info(lexicon.size() + " features in the lexicon"); lexicon.save(manager.getLexiconFileName()); }