opennlp.tools.ml.model.DataIndexer.index java code examples

Common ways to obtain DataIndexer

private void myMethod () {
 DataIndexer d =new OnePassDataIndexer()
TrainingParameters parameters;Map reportMap;DataIndexerFactory.getDataIndexer(parameters, reportMap)
Smart code suggestions by Codota
}

public DataIndexer getDataIndexer(ObjectStream<Event> events) throws IOException {
 trainingParameters.put(AbstractDataIndexer.SORT_PARAM, isSortAndMerge());
 // If the cutoff was set, don't overwrite the value.
 if (trainingParameters.getIntParameter(CUTOFF_PARAM, -1) == -1) {
  trainingParameters.put(CUTOFF_PARAM, 5);
 }
 
 DataIndexer indexer = DataIndexerFactory.getDataIndexer(trainingParameters, reportMap);
 indexer.index(events);
 return indexer;
}

 @Test
 public void testLastLineBug() throws IOException {
  try (RealValueFileEventStream rvfes = new RealValueFileEventStream(
    "src/test/resources/data/opennlp/maxent/io/rvfes-bug-data-ok.txt")) {
   indexer.index(rvfes);
  }
  Assert.assertEquals(1, indexer.getOutcomeLabels().length);

  try (RealValueFileEventStream rvfes = new RealValueFileEventStream(
    "src/test/resources/data/opennlp/maxent/io/rvfes-bug-data-broken.txt")) {
   indexer.index(rvfes);
  }
  Assert.assertEquals(1, indexer.getOutcomeLabels().length);
 }
}

@Test
public void testTrainModelReturnsAQNModel() throws Exception {
 // given
 RealValueFileEventStream rvfes1 = new RealValueFileEventStream(
   "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt");
 testDataIndexer.index(rvfes1);
 // when
 QNModel trainedModel = new QNTrainer(false).trainModel(ITERATIONS, testDataIndexer);
 // then
 Assert.assertNotNull(trainedModel);
}

@Test
public void testValueAtNonInitialPoint01() throws IOException {
 // given
 RealValueFileEventStream rvfes1 = new RealValueFileEventStream(
   "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8");
 testDataIndexer.index(rvfes1);
 NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer);
 // when
 double[] nonInitialPoint = new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
 double value = objectFunction.valueAt(nonInitialPoint);
 double expectedValue = 13.862943611198894;
 // then
 Assert.assertEquals(expectedValue, value, TOLERANCE01);
}

@Test
public void testMaxentOnPrepAttachData() throws IOException {
 testDataIndexer.index(PrepAttachDataUtil.createTrainingStream());
 // this shows why the GISTrainer should be a AbstractEventTrainer.
 // TODO: make sure that the trainingParameter cutoff and the 
 // cutoff value passed here are equal.
 AbstractModel model =
   new GISTrainer(true).trainModel(100,
   testDataIndexer,
   new UniformPrior(), 1);
 PrepAttachDataUtil.testModel(model, 0.7997028967566229);
}

@Test
public void testNaiveBayes4() throws IOException {
 testDataIndexer.index(createTrainingStream());
 NaiveBayesModel model =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 String label = "politics";
 String[] context = {};
 Event event = new Event(label, context);
 testModel(model, event, 7.0 / 12.0);
}

@Test
public void testNaiveBayes2() throws IOException {
 testDataIndexer.index(createTrainingStream());
 NaiveBayesModel model =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 String label = "sports";
 String[] context = {"bow=manchester", "bow=united"};
 Event event = new Event(label, context);
 // testModel(model, event, 1.0);  // Expected value without smoothing
 testModel(model, event, 0.9658833555831029);   // Expected value with smoothing
}

@Test
public void testMaxentOnPrepAttachData2Threads() throws IOException {
 testDataIndexer.index(PrepAttachDataUtil.createTrainingStream());
 AbstractModel model =
   new GISTrainer(true).trainModel(100,
     testDataIndexer,
     new UniformPrior(), 2);
 PrepAttachDataUtil.testModel(model, 0.7997028967566229);
}

@Test
public void testNaiveBayes3() throws IOException {
 testDataIndexer.index(createTrainingStream());
 NaiveBayesModel model =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 String label = "politics";
 String[] context = {"bow=united"};
 Event event = new Event(label, context);
 //testModel(model, event, 2.0/3.0);  // Expected value without smoothing
 testModel(model, event, 0.6655036407766989);  // Expected value with smoothing
}

@Test
public void testNaiveBayesOnPrepAttachData() throws IOException {
 testDataIndexer.index(PrepAttachDataUtil.createTrainingStream());
 MaxentModel model = new NaiveBayesTrainer().trainModel(testDataIndexer);
 Assert.assertTrue(model instanceof NaiveBayesModel);
 PrepAttachDataUtil.testModel(model, 0.7897994553107205);
}

@Test
public void testNaiveBayes1() throws IOException {
 testDataIndexer.index(createTrainingStream());
 NaiveBayesModel model =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 String label = "politics";
 String[] context = {"bow=united", "bow=nations"};
 Event event = new Event(label, context);
 // testModel(model, event, 1.0);  // Expected value without smoothing
 testModel(model, event, 0.9681650180264167);   // Expected value with smoothing
}

@Test
public void testModel() throws IOException {
 // given
 RealValueFileEventStream rvfes1 = new RealValueFileEventStream(
   "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt");
 testDataIndexer.index(rvfes1);
 // when
 QNModel trainedModel = new QNTrainer(15, true).trainModel(
   ITERATIONS, testDataIndexer);
 Assert.assertFalse(trainedModel.equals(null));
}

@Test
public void testValueAtInitialPoint() throws IOException {
 // given
 RealValueFileEventStream rvfes1 = new RealValueFileEventStream(
   "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8");
 testDataIndexer.index(rvfes1);
 NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer);
 // when
 double value = objectFunction.valueAt(objectFunction.getInitialPoint());
 double expectedValue = 13.86294361;
 // then
 Assert.assertEquals(expectedValue, value, TOLERANCE01);
}

@Test
public void testGradientSanity() throws IOException {
 // given
 RealValueFileEventStream rvfes1 = new RealValueFileEventStream(
   "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8");
 testDataIndexer.index(rvfes1);
 NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer);
 // when
 double[] initial = objectFunction.getInitialPoint();
 double[] gradientAtInitial = objectFunction.gradientAt(initial);
 // then
 Assert.assertNotNull(gradientAtInitial);
}

@Test
public void testNaiveBayes1() throws IOException {
 testDataIndexer.index(NaiveBayesCorrectnessTest.createTrainingStream());
 NaiveBayesModel model1 =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 NaiveBayesModel model2 = persistedModel(model1);
 String label = "politics";
 String[] context = {"bow=united", "bow=nations"};
 Event event = new Event(label, context);
 testModelOutcome(model1, model2, event);
}

@Test
public void testQNOnPrepAttachData() throws IOException {
 DataIndexer indexer = new TwoPassDataIndexer();
 TrainingParameters indexingParameters = new TrainingParameters();
 indexingParameters.put(AbstractTrainer.CUTOFF_PARAM, 1);
 indexingParameters.put(AbstractDataIndexer.SORT_PARAM, false);
 indexer.init(indexingParameters, new HashMap<>());
 indexer.index(PrepAttachDataUtil.createTrainingStream());
 AbstractModel model = new QNTrainer(true).trainModel(100, indexer );
 PrepAttachDataUtil.testModel(model, 0.8155484030700668);
}

@Test
public void testNaiveBayes2() throws IOException {
 testDataIndexer.index(NaiveBayesCorrectnessTest.createTrainingStream());
 NaiveBayesModel model1 =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 NaiveBayesModel model2 = persistedModel(model1);
 String label = "sports";
 String[] context = {"bow=manchester", "bow=united"};
 Event event = new Event(label, context);
 testModelOutcome(model1, model2, event);
}

@Test
public void testNaiveBayes3() throws IOException {
 testDataIndexer.index(NaiveBayesCorrectnessTest.createTrainingStream());
 NaiveBayesModel model1 =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 NaiveBayesModel model2 = persistedModel(model1);
 String label = "politics";
 String[] context = {"bow=united"};
 Event event = new Event(label, context);
 testModelOutcome(model1, model2, event);
}

@Test
public void testNaiveBayes4() throws IOException {
 testDataIndexer.index(NaiveBayesCorrectnessTest.createTrainingStream());
 NaiveBayesModel model1 =
   (NaiveBayesModel) new NaiveBayesTrainer().trainModel(testDataIndexer);
 NaiveBayesModel model2 = persistedModel(model1);
 String label = "politics";
 String[] context = {};
 Event event = new Event(label, context);
 testModelOutcome(model1, model2, event);
}

@Test
public void testDomainDimensionSanity() throws IOException {
 // given
 RealValueFileEventStream rvfes1 = new RealValueFileEventStream(
   "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8");
 testDataIndexer.index(rvfes1);
 NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer);
 // when
 int correctDomainDimension = testDataIndexer.getPredLabels().length
   * testDataIndexer.getOutcomeLabels().length;
 // then
 Assert.assertEquals(correctDomainDimension, objectFunction.getDimension());
}

Javadoc

Performs the data indexing. Make sure the init(...) method is called first.

Popular methods of DataIndexer

getContexts
Returns the array of predicates seen in each event.
getNumEvents
Returns the number of total events indexed.
getNumTimesEventsSeen
Returns an array indicating the number of times a particular event was seen.
getOutcomeLabels
Returns an array of outcome names.
getOutcomeList
Returns an array indicating the outcome index for each event.
getPredCounts
Returns an array of the count of each predicate in the events.
getPredLabels
Returns an array of predicate/context names.
getValues
Returns the values associated with each event context or null if integer values are to be used.
init
Sets parameters used during the data indexing.

Popular in Java

Reading from database using SQL prepared statement
addToBackStack (FragmentTransaction)
putExtra (Intent)
scheduleAtFixedRate (ScheduledExecutorService)
Creates and executes a periodic action that becomes enabled first after the given initial delay, and
FileNotFoundException (java.io)
Thrown when a file specified by a program cannot be found.
Socket (java.net)
Provides a client-side TCP socket.
SimpleDateFormat (java.text)
Formats and parses dates in a locale-sensitive manner. Formatting turns a Date into a String, and pa
CountDownLatch (java.util.concurrent)
A synchronization aid that allows one or more threads to wait until a set of operations being perfor
DataSource (javax.sql)
A factory for connections to the physical data source that this DataSource object represents. An alt
JComboBox (javax.swing)

How to use indexmethodin opennlp.tools.ml.model.DataIndexer

Best Java code snippets using opennlp.tools.ml.model.DataIndexer.index (Showing top 20 results out of 315)

How to use
index
method
in
opennlp.tools.ml.model.DataIndexer