@Test public void testLoadConfusionSet() throws IOException { try (InputStream inputStream = JLanguageTool.getDataBroker().getFromResourceDirAsStream("/yy/neuralnetwork_confusion_sets.txt")) { List<ScoredConfusionSet> list = ScoredConfusionSetLoader.loadConfusionSet(inputStream); assertThat(list.size(), is(6)); assertThat(list.get(0).getConfusionTokens().size(), is(2)); assertThat(list.get(0).getConfusionTokens().get(0), is("their")); assertThat(list.get(0).getConfusionTokens().get(1), is("there")); assertThat(list.get(0).getTokenDescriptions().size(), is(2)); assertThat(list.get(0).getTokenDescriptions().get(0).orElse("fail"), is("example 2")); assertThat(list.get(0).getTokenDescriptions().get(1).orElse("fail"), is("example 1")); assertThat(list.get(1).getTokenDescriptions().get(1).orElse("ok"), is("ok")); assertThat(list.get(0).getScore(), is(5.0f)); assertThat(list.get(1).getScore(), is(1.1f)); assertThat(list.get(2).getScore(), is(0.5f)); assertThat(list.get(3).getScore(), is(0.8f)); assertThat(list.get(4).getScore(), is(1.2f)); assertThat(list.get(5).getScore(), is(1.0f)); assertThat(list.get(5).getConfusionTokens().get(0), is("im")); } }
public NeuralNetworkRule(ResourceBundle messages, Language language, ScoredConfusionSet confusionSet, Classifier classifier) { super(messages); super.setCategory(Categories.TYPOS.getCategory(messages)); this.subjects = confusionSet.getConfusionTokens(); this.descriptions = confusionSet.getTokenDescriptions(); this.minScore = confusionSet.getScore(); this.classifier = classifier; this.id = createId(language); }
public NeuralNetworkRule(ResourceBundle messages, Language language, ScoredConfusionSet confusionSet, Word2VecModel word2VecModel) throws IOException { super(messages); super.setCategory(Categories.TYPOS.getCategory(messages)); this.subjects = confusionSet.getConfusionTokens(); this.descriptions = confusionSet.getTokenDescriptions(); this.minScore = confusionSet.getScore(); try { InputStream W1Stream = streamFor(word2VecModel.getPath(), "W_fc1.txt"); InputStream b1Stream = streamFor(word2VecModel.getPath(), "b_fc1.txt"); Classifier tmpClassifier; try { InputStream W2Stream = streamFor(word2VecModel.getPath(), "W_fc2.txt"); InputStream b2Stream = streamFor(word2VecModel.getPath(), "b_fc2.txt"); //System.out.println("deep rule for " + confusionSet.toString()); tmpClassifier = new TwoLayerClassifier(word2VecModel.getEmbedding(), W1Stream, b1Stream, W2Stream, b2Stream); } catch (FileNotFoundException e) { tmpClassifier = new SingleLayerClassifier(word2VecModel.getEmbedding(), W1Stream, b1Stream); } classifier = tmpClassifier; } catch (FileNotFoundException e) { throw new IOException("Weights for confusion set " + confusionSet.toString() + " are missing", e); } this.id = createId(language); }
public NeuralNetworkRule(ResourceBundle messages, Language language, ScoredConfusionSet confusionSet, Classifier classifier) { super(messages); super.setCategory(Categories.TYPOS.getCategory(messages)); this.subjects = confusionSet.getConfusionTokens(); this.descriptions = confusionSet.getTokenDescriptions(); this.minScore = confusionSet.getScore(); this.classifier = classifier; this.id = createId(language); }
public NeuralNetworkRule(ResourceBundle messages, Language language, ScoredConfusionSet confusionSet, Word2VecModel word2VecModel) throws IOException { super(messages); super.setCategory(Categories.TYPOS.getCategory(messages)); this.subjects = confusionSet.getConfusionTokens(); this.descriptions = confusionSet.getTokenDescriptions(); this.minScore = confusionSet.getScore(); try { InputStream W1Stream = streamFor(word2VecModel.getPath(), "W_fc1.txt"); InputStream b1Stream = streamFor(word2VecModel.getPath(), "b_fc1.txt"); Classifier tmpClassifier; try { InputStream W2Stream = streamFor(word2VecModel.getPath(), "W_fc2.txt"); InputStream b2Stream = streamFor(word2VecModel.getPath(), "b_fc2.txt"); //System.out.println("deep rule for " + confusionSet.toString()); tmpClassifier = new TwoLayerClassifier(word2VecModel.getEmbedding(), W1Stream, b1Stream, W2Stream, b2Stream); } catch (FileNotFoundException e) { tmpClassifier = new SingleLayerClassifier(word2VecModel.getEmbedding(), W1Stream, b1Stream); } classifier = tmpClassifier; } catch (FileNotFoundException e) { throw new IOException("Weights for confusion set " + confusionSet.toString() + " are missing", e); } this.id = createId(language); }