public final MaxentModel train(ObjectStream<Event> events) throws IOException { validate(); HashSumEventStream hses = new HashSumEventStream(events); DataIndexer indexer = getDataIndexer(hses); addToReport("Training-Eventhash", hses.calculateHashSum().toString(16)); return train(indexer); } }
int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); // do not close. try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum();
public final MaxentModel train(ObjectStream<Event> events) throws IOException { validate(); HashSumEventStream hses = new HashSumEventStream(events); DataIndexer indexer = getDataIndexer(hses); addToReport("Training-Eventhash", hses.calculateHashSum().toString(16)); return train(indexer); } }
public final MaxentModel train(ObjectStream<Event> events) throws IOException { validate(); HashSumEventStream hses = new HashSumEventStream(events); DataIndexer indexer = getDataIndexer(hses); addToReport("Training-Eventhash", hses.calculateHashSum().toString(16)); return train(indexer); } }
int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); // do not close. try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum();
public static SentenceModel train(String languageCode, ObjectStream<SentenceSample> samples, boolean useTokenEnd, Dictionary abbreviations, int cutoff, int iterations) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<String, String>(); ModelUtil.addCutoffAndIterations(manifestInfoEntries, cutoff, iterations); Factory factory = new Factory(); // TODO: Fix the EventStream to throw exceptions when training goes wrong SDEventStream eventStream = new SDEventStream(samples, factory.createSentenceContextGenerator(languageCode), factory.createEndOfSentenceScanner(languageCode)); HashSumEventStream hses = new HashSumEventStream(eventStream); // AbstractObjectStream<Event> GISTrainer trainer = new GISTrainer(); MaxentModel sentModel = trainer.trainModel(hses, iterations, cutoff); manifestInfoEntries.put(BaseModel.TRAINING_EVENTHASH_PROPERTY, hses.calculateHashSum().toString(16)); return new SentenceModel(languageCode, sentModel, useTokenEnd, abbreviations, manifestInfoEntries); }
public static SentenceModel train(String languageCode, ObjectStream<SentenceSample> samples, boolean useTokenEnd, Dictionary abbreviations, int cutoff, int iterations) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<String, String>(); ModelUtil.addCutoffAndIterations(manifestInfoEntries, cutoff, iterations); Factory factory = new Factory(); // TODO: Fix the EventStream to throw exceptions when training goes wrong SDEventStream eventStream = new SDEventStream(samples, factory.createSentenceContextGenerator(languageCode), factory.createEndOfSentenceScanner(languageCode)); HashSumEventStream hses = new HashSumEventStream(eventStream); // AbstractObjectStream<Event> GISTrainer trainer = new GISTrainer(); MaxentModel sentModel = trainer.trainModel(hses, iterations, cutoff); manifestInfoEntries.put(BaseModel.TRAINING_EVENTHASH_PROPERTY, hses.calculateHashSum().toString(16)); return new SentenceModel(languageCode, sentModel, useTokenEnd, abbreviations, manifestInfoEntries); }