public ParserModel updateCheckModel(MaxentModel checkModel) { return new ParserModel(getLanguage(), getBuildModel(), checkModel, getAttachModel(), getParserTaggerModel(), getParserChunkerModel(), getHeadRules(), getParserType()); }
public ParserModel(String languageCode, MaxentModel buildModel, MaxentModel checkModel, MaxentModel attachModel, POSModel parserTagger, ChunkerModel chunkerTagger, opennlp.tools.parser.HeadRules headRules, ParserType modelType, Map<String, String> manifestInfoEntries) { super(COMPONENT_NAME, languageCode, manifestInfoEntries); setManifestProperty(PARSER_TYPE, modelType.name()); artifactMap.put(BUILD_MODEL_ENTRY_NAME, buildModel); artifactMap.put(CHECK_MODEL_ENTRY_NAME, checkModel); if (ParserType.CHUNKING.equals(modelType)) { if (attachModel != null) throw new IllegalArgumentException("attachModel must be null for chunking parser!"); } else if (ParserType.TREEINSERT.equals(modelType)) { Objects.requireNonNull(attachModel, "attachModel must not be null"); artifactMap.put(ATTACH_MODEL_ENTRY_NAME, attachModel); } else { throw new IllegalStateException("Unknown ParserType '" + modelType + "'!"); } artifactMap.put(PARSER_TAGGER_MODEL_ENTRY_NAME, parserTagger); artifactMap.put(CHUNKER_TAGGER_MODEL_ENTRY_NAME, chunkerTagger); artifactMap.put(HEAD_RULES_MODEL_ENTRY_NAME, headRules); checkArtifactMap(); }
public ParserModel updateChunkerModel(ChunkerModel chunkModel) { return new ParserModel(getLanguage(), getBuildModel(), getCheckModel(), getAttachModel(), getParserTaggerModel(), chunkModel, getHeadRules(), getParserType()); }
public ParserModel updateBuildModel(MaxentModel buildModel) { return new ParserModel(getLanguage(), buildModel, getCheckModel(), getAttachModel(), getParserTaggerModel(), getParserChunkerModel(), getHeadRules(), getParserType()); }
public ParserModel updateTaggerModel(POSModel taggerModel) { return new ParserModel(getLanguage(), getBuildModel(), getCheckModel(), getAttachModel(), taggerModel, getParserChunkerModel(), getHeadRules(), getParserType()); }
@Override protected ParserModel loadModel(InputStream modelIn) throws IOException, InvalidFormatException { return new ParserModel(modelIn); }
public Parser(ParserModel model, int beamSize, double advancePercentage) { this(model.getBuildModel(), model.getAttachModel(), model.getCheckModel(), new POSTaggerME(model.getParserTaggerModel()), new ChunkerME(model.getParserChunkerModel()), model.getHeadRules(), beamSize, advancePercentage); }
@Override protected Parser produceResource(InputStream aStream) throws Exception { ParserModel model = new ParserModel(aStream); Properties metadata = getResourceMetaData(); addTagset(new OpenNlpTagsetDescriptionProvider( metadata.getProperty("pos.tagset"), POS.class, model.getParserTaggerModel() .getPosModel())); addTagset(new OpenNlpParserTagsetDescriptionProvider( metadata.getProperty("constituent.tagset"), Constituent.class, model, metadata)); if (printTagSet) { getContext().getLogger().log(INFO, getTagset().toString()); } return ParserFactory.create(model); } }
model.serialize(outArray); outArray.close(); ParserModel outputModel = new ParserModel(new ByteArrayInputStream(outArray.toByteArray()));
@Override public Set<String> listTags(String aLayer, String aTagsetName) { Set<String> tagSet = new TreeSet<String>(); SequenceClassificationModel<TokenTag> seqModel = model.getParserChunkerModel() .getChunkerSequenceModel(); collect(seqModel.getOutcomes(), tagSet); if (model.getBuildModel() != null) { collect(model.getBuildModel(), tagSet); } return tagSet; }
public ParserModel updateBuildModel(MaxentModel buildModel) { return new ParserModel(getLanguage(), buildModel, getCheckModel(), getAttachModel(), getParserTaggerModel(), getParserChunkerModel(), getHeadRules(), getParserType()); }
public ParserModel updateTaggerModel(POSModel taggerModel) { return new ParserModel(getLanguage(), getBuildModel(), getCheckModel(), getAttachModel(), taggerModel, getParserChunkerModel(), getHeadRules(), getParserType()); }
@Test public void evalParserModel() throws Exception { ParserModel model = new ParserModel( new File(getOpennlpDataDir(), "models-sf/en-parser-chunking.bin")); MessageDigest digest = MessageDigest.getInstance(HASH_ALGORITHM); Parser parser = ParserFactory.create(model); try (ObjectStream<LeipzigTestSample> lines = createLineWiseStream()) { LeipzigTestSample line; while ((line = lines.read()) != null) { Parse[] parse = ParserTool.parseLine(String.join(" ", line.getText()), parser, 1); if (parse.length > 0) { StringBuffer sb = new StringBuffer(); parse[0].show(sb); digest.update(sb.toString().getBytes(StandardCharsets.UTF_8)); } else { digest.update("empty".getBytes(StandardCharsets.UTF_8)); } } } Assert.assertEquals(new BigInteger("312218841713337505306598301082074515847"), new BigInteger(1, digest.digest())); } }
public Parser(ParserModel model, int beamSize, double advancePercentage) { this(model.getBuildModel(), model.getCheckModel(), new POSTaggerME(model.getParserTaggerModel()), new ChunkerME(model.getParserChunkerModel()), model.getHeadRules(), beamSize, advancePercentage); }
/** * Verify that training and tagging does not cause * runtime problems. */ @Test public void testTreeInsertParserTraining() throws Exception { ObjectStream<Parse> parseSamples = ParserTestUtil.openTestTrainingData(); HeadRules headRules = ParserTestUtil.createTestHeadRules(); ParserModel model = Parser.train("eng", parseSamples, headRules, 100, 0); opennlp.tools.parser.Parser parser = ParserFactory.create(model); // Tests parsing to make sure the code does not has // a bug which fails always with a runtime exception parser.parse(Parse.parseParse("She was just another freighter from the " + "States and she seemed as commonplace as her name .")); // Test serializing and de-serializing model ByteArrayOutputStream outArray = new ByteArrayOutputStream(); model.serialize(outArray); outArray.close(); new ParserModel(new ByteArrayInputStream(outArray.toByteArray())); // TODO: compare both models } }
public ParserModel updateCheckModel(MaxentModel checkModel) { return new ParserModel(getLanguage(), getBuildModel(), checkModel, getAttachModel(), getParserTaggerModel(), getParserChunkerModel(), getHeadRules(), getParserType()); }
public ParserModel updateChunkerModel(ChunkerModel chunkModel) { return new ParserModel(getLanguage(), getBuildModel(), getCheckModel(), getAttachModel(), getParserTaggerModel(), chunkModel, getHeadRules(), getParserType()); }
public ParserModel updateBuildModel(MaxentModel buildModel) { return new ParserModel(getLanguage(), buildModel, getCheckModel(), getAttachModel(), getParserTaggerModel(), getParserChunkerModel(), getHeadRules(), getParserType()); }
public ParserModel updateTaggerModel(POSModel taggerModel) { return new ParserModel(getLanguage(), getBuildModel(), getCheckModel(), getAttachModel(), taggerModel, getParserChunkerModel(), getHeadRules(), getParserType()); }
@Override protected ParserModel loadModel(InputStream modelIn) throws IOException, InvalidFormatException { return new ParserModel(modelIn); }