@Override public void write(Instance<OUTCOME_TYPE> instance) throws CleartkProcessingException { this.trainingDataWriter.println(instance.getOutcome()); }
@Override public void write(Instance<String> instance) throws CleartkProcessingException { if(instance.getOutcome().startsWith("#DEBUG")){ this.trainingDataWriter.println(instance.getOutcome()); this.trainingDataWriter.flush(); }else{ super.write(instance); } } }
public void write(List<Instance<OUTCOME_TYPE>> instances) throws CleartkProcessingException { if (this.delegatedDataWriter == null) throw new IllegalStateException( "delegatedDataWriter must be set before calling writeSequence"); List<Object> outcomes = new ArrayList<Object>(); for (Instance<OUTCOME_TYPE> instance : instances) { List<Feature> instanceFeatures = instance.getFeatures(); for (OutcomeFeatureExtractor outcomeFeatureExtractor : outcomeFeatureExtractors) { instanceFeatures.addAll(outcomeFeatureExtractor.extractFeatures(outcomes)); } outcomes.add(instance.getOutcome()); delegatedDataWriter.write(instance); } }
@Override public Instance<OUTCOME_T> transform(Instance<OUTCOME_T> instance) { List<Feature> features = new ArrayList<Feature>(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { // Filter down to selected features features.addAll(Collections2.filter(((TransformableFeature) feature).getFeatures(), this)); } else { // Pass non-relevant features through w/o filtering features.add(feature); } } return new Instance<OUTCOME_T>(instance.getOutcome(), features); }
@Override public Instance<OUTCOME_T> transform(Instance<OUTCOME_T> instance) { List<Feature> features = new ArrayList<Feature>(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { // Filter down to selected features features.addAll(Collections2.filter(((TransformableFeature) feature).getFeatures(), this)); } else { // Pass non-relevant features through w/o filtering features.add(feature); } } return new Instance<OUTCOME_T>(instance.getOutcome(), features); }
@Override public Instance<OUTCOME_T> transform(Instance<OUTCOME_T> instance) { List<Feature> features = new ArrayList<Feature>(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { // Filter down to selected features features.addAll(Collections2.filter(((TransformableFeature) feature).getFeatures(), this)); } else { // Pass non-relevant features through w/o filtering features.add(feature); } } return new Instance<OUTCOME_T>(instance.getOutcome(), features); }
@Override public Instance<OUTCOME_T> transform(Instance<OUTCOME_T> instance) { List<Feature> features = new ArrayList<Feature>(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { for (Feature origFeature : ((TransformableFeature) feature).getFeatures()) { features.add(this.transform(origFeature)); } } else { features.add(feature); } } return new Instance<OUTCOME_T>(instance.getOutcome(), features); }
@Override public Instance<OUTCOME_T> transform(Instance<OUTCOME_T> instance) { List<Feature> features = new ArrayList<Feature>(); List<Feature> featuresToTransform = new ArrayList<Feature>(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { // Store off features for later similarity computation featuresToTransform.addAll(((TransformableFeature) feature).getFeatures()); } else { // pass through non-transformable features features.add(feature); } } // Create centroid similarity feature Map<String, Double> featureMap = this.featuresToFeatureMap(featuresToTransform); features.add(new Feature(this.name, new Double(this.simFunction.distance( featureMap, centroidMap)))); return new Instance<OUTCOME_T>(instance.getOutcome(), features); }
@Override public void write(Instance<OUTCOME_TYPE> instance) throws CleartkProcessingException { if (instance.getOutcome() == null) { throw CleartkProcessingException.noInstanceOutcome(instance.getFeatures()); } String outcome = this.classifierBuilder.getOutcomeEncoder().encode(instance.getOutcome()); ContextValues contextValues = this.classifierBuilder.getFeaturesEncoder().encodeAll(instance.getFeatures()); this.trainingDataWriter.printf("%s %s\n", outcome, contextValues.toMaxentString()); } }
public void write(Instance<OUTCOME_TYPE> instance) throws CleartkProcessingException { writeEncoded( this.classifierBuilder.getFeaturesEncoder().encodeAll(instance.getFeatures()), this.classifierBuilder.getOutcomeEncoder().encode(instance.getOutcome())); }
@Override public void train(Iterable<Instance<OUTCOME_T>> instances) { // aggregate statistics for all features this.chi2Function = new Chi2Scorer<OUTCOME_T>(this.yates); for (Instance<OUTCOME_T> instance : instances) { OUTCOME_T outcome = instance.getOutcome(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { for (Feature untransformedFeature : ((TransformableFeature) feature).getFeatures()) { this.chi2Function.update(this.getFeatureName(untransformedFeature), outcome, 1); } } } } // keep only large chi2 valued features this.selectedFeatureNames = Sets.newHashSet(); for (String featureName : this.chi2Function.featValueClassCount.rowKeySet()) { if (this.chi2Function.score(featureName) > this.chi2Threshold) { this.selectedFeatureNames.add(featureName); } } this.isTrained = true; }
@Override public void train(Iterable<Instance<OUTCOME_T>> instances) { // aggregate statistics for all features this.chi2Function = new Chi2Scorer<OUTCOME_T>(this.yates); for (Instance<OUTCOME_T> instance : instances) { OUTCOME_T outcome = instance.getOutcome(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { for (Feature untransformedFeature : ((TransformableFeature) feature).getFeatures()) { this.chi2Function.update(this.getFeatureName(untransformedFeature), outcome, 1); } } } } // keep only large chi2 valued features this.selectedFeatureNames = Sets.newHashSet(); for (String featureName : this.chi2Function.featValueClassCount.rowKeySet()) { if (this.chi2Function.score(featureName) > this.chi2Threshold) { this.selectedFeatureNames.add(featureName); } } this.isTrained = true; }
public void write(List<Instance<OUTCOME_TYPE>> instances) throws CleartkProcessingException { for (Instance<OUTCOME_TYPE> instance : instances) { writeEncoded( this.classifierBuilder.getFeaturesEncoder().encodeAll(instance.getFeatures()), this.classifierBuilder.getOutcomeEncoder().encode(instance.getOutcome())); } this.writeEndSequence(); }
@Override public void write(Instance<Double> instance) throws CleartkProcessingException { if (!(instance instanceof QidInstance)) { throw new CleartkProcessingException("", "Unable to write non-QidInstance"); } String qid = ((QidInstance<Double>) instance).getQid(); writeEncoded( this.getEncodedQid(qid), this.classifierBuilder.getFeaturesEncoder().encodeAll(instance.getFeatures()), this.classifierBuilder.getOutcomeEncoder().encode(instance.getOutcome())); }
@Override public void write(List<Instance<OUTCOME_TYPE>> instances) throws CleartkProcessingException { try { for (Instance<OUTCOME_TYPE> instance : instances) { writeEncoded( this.classifierBuilder.getFeaturesEncoder().encodeAll(instance.getFeatures()), this.classifierBuilder.getOutcomeEncoder().encode(instance.getOutcome())); } writeSequenceEnd(); } catch (IOException e) { throw new CleartkProcessingException(e); } }
@Override public void write(Instance<Double> instance) throws CleartkProcessingException { if (!(instance instanceof QidInstance)) { throw new CleartkProcessingException("", "Unable to write non-QidInstance"); } String qid = ((QidInstance<Double>) instance).getQid(); writeEncoded( this.getEncodedQid(qid), this.classifierBuilder.getFeaturesEncoder().encodeAll(instance.getFeatures()), this.classifierBuilder.getOutcomeEncoder().encode(instance.getOutcome())); }
@Override public void train(Iterable<Instance<OUTCOME_T>> instances) { // aggregate statistics for all features and classes this.mutualInfoStats = new MutualInformationStats<OUTCOME_T>(this.smoothingCount); for (Instance<OUTCOME_T> instance : instances) { OUTCOME_T outcome = instance.getOutcome(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { for (Feature untransformedFeature : ((TransformableFeature) feature).getFeatures()) { mutualInfoStats.update(this.getFeatureName(untransformedFeature), outcome, 1); } } } } // sort features by mutual information score Set<String> featureNames = mutualInfoStats.classConditionalCounts.rowKeySet(); Function<String, Double> scoreFunction = this.mutualInfoStats.getScoreFunction(this.combineScoreMethod); Ordering<String> ordering = Ordering.natural().onResultOf(scoreFunction).reverse(); // keep only the top N features this.selectedFeatureNames = Sets.newLinkedHashSet(ordering.immutableSortedCopy(featureNames).subList( 0, this.numFeatures)); this.isTrained = true; }
@Override public void train(Iterable<Instance<OUTCOME_T>> instances) { // aggregate statistics for all features and classes this.mutualInfoStats = new MutualInformationStats<OUTCOME_T>(this.smoothingCount); for (Instance<OUTCOME_T> instance : instances) { OUTCOME_T outcome = instance.getOutcome(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { for (Feature untransformedFeature : ((TransformableFeature) feature).getFeatures()) { mutualInfoStats.update(this.getFeatureName(untransformedFeature), outcome, 1); } } } } // sort features by mutual information score Set<String> featureNames = mutualInfoStats.classConditionalCounts.rowKeySet(); Function<String, Double> scoreFunction = this.mutualInfoStats.getScoreFunction(this.combineScoreMethod); Ordering<String> ordering = Ordering.natural().onResultOf(scoreFunction).reverse(); // keep only the top N features this.selectedFeatureNames = Sets.newLinkedHashSet(ordering.immutableSortedCopy(featureNames).subList( 0, this.numFeatures)); this.isTrained = true; }
@Override public void train(Iterable<Instance<OUTCOME_T>> instances) { // aggregate statistics for all features and classes this.mutualInfoStats = new MutualInformationStats<OUTCOME_T>(this.smoothingCount); for (Instance<OUTCOME_T> instance : instances) { OUTCOME_T outcome = instance.getOutcome(); for (Feature feature : instance.getFeatures()) { if (this.isTransformable(feature)) { for (Feature untransformedFeature : ((TransformableFeature) feature).getFeatures()) { mutualInfoStats.update(this.nameFeature(untransformedFeature), outcome, 1); } } } } // Compute mutual information score for each feature Set<String> featureNames = mutualInfoStats.classConditionalCounts.rowKeySet(); this.selectedFeatures = Ordering.natural().onResultOf( this.mutualInfoStats.getScoreFunction( this.combineScoreMethod)).reverse().immutableSortedCopy(featureNames); this.isTrained = true; }
if (instance.getOutcome()!=null) { if(coin.nextDouble() < this.portionOfDataToUse){ this.dataWriter.write(new Instance<>(instance.getOutcome(),feats));