@Override public PipelineModelInfo getModelInfo(final PipelineModel from) { final PipelineModelInfo modelInfo = new PipelineModelInfo(); final ModelInfo stages[] = new ModelInfo[from.stages().length]; for (int i = 0; i < from.stages().length; i++) { Transformer sparkModel = from.stages()[i]; stages[i] = ModelInfoAdapterFactory.getAdapter(sparkModel.getClass()).adapt(sparkModel); } modelInfo.setStages(stages); return modelInfo; }
@Override public PipelineModelInfo getModelInfo(final PipelineModel from, final DataFrame df) { final PipelineModelInfo modelInfo = new PipelineModelInfo(); final ModelInfo stages[] = new ModelInfo[from.stages().length]; for (int i = 0; i < from.stages().length; i++) { Transformer sparkModel = from.stages()[i]; stages[i] = ModelInfoAdapterFactory.getAdapter(sparkModel.getClass()).adapt(sparkModel, df); } modelInfo.setStages(stages); return modelInfo; }
/** * Creates a conditional Markov model. * @param pipelineModel * @param weights * @param markovOrder */ public CMMModel(PipelineModel pipelineModel, Vector weights, MarkovOrder markovOrder, Map<String, Set<Integer>> tagDictionary) { this.pipelineModel = pipelineModel; this.contextExtractor = new ContextExtractor(markovOrder, Constants.REGEXP_FILE); this.weights = weights; this.tags = ((StringIndexerModel)(pipelineModel.stages()[2])).labels(); String[] features = ((CountVectorizerModel)(pipelineModel.stages()[1])).vocabulary(); featureMap = new HashMap<String, Integer>(); for (int j = 0; j < features.length; j++) { featureMap.put(features[j], j); } this.tagDictionary = tagDictionary; }
@Override public List<Transformer> apply(Transformer transformer){ if(transformer instanceof PipelineModel){ PipelineModel pipelineModel = (PipelineModel)transformer; return Arrays.asList(pipelineModel.stages()); } else if(transformer instanceof CrossValidatorModel){ CrossValidatorModel crossValidatorModel = (CrossValidatorModel)transformer; return Collections.singletonList(crossValidatorModel.bestModel()); } else if(transformer instanceof TrainValidationSplitModel){ TrainValidationSplitModel trainValidationSplitModel = (TrainValidationSplitModel)transformer; return Collections.singletonList(trainValidationSplitModel.bestModel()); } return null; } };
/** * Creates a transition-based parser using a MLP transition classifier. * @param jsc * @param classifierFileName * @param featureFrame */ public TransitionBasedParserMLP(JavaSparkContext jsc, String classifierFileName, FeatureFrame featureFrame) { this.featureFrame = featureFrame; this.classifier = TransitionClassifier.load(jsc, new Path(classifierFileName, "data").toString()); this.pipelineModel = PipelineModel.load(new Path(classifierFileName, "pipelineModel").toString()); this.transitionName = ((StringIndexerModel)pipelineModel.stages()[2]).labels(); String[] features = ((CountVectorizerModel)(pipelineModel.stages()[1])).vocabulary(); this.featureMap = new HashMap<String, Integer>(); for (int j = 0; j < features.length; j++) { this.featureMap.put(features[j], j); } }
public void printModel() { LogisticRegressionModel lrModel = (LogisticRegressionModel) model.stages()[2]; System.out.println("intercept = " + lrModel.intercept()); System.out.println("number of features = " + lrModel.numFeatures()); System.out.println("regularization parameter = " + lrModel.getRegParam()); System.out.println(lrModel.explainParams()); }
Transformer[] stages = pipelineModel.stages(); for(Transformer stage : stages){ TransformerConverter<?> converter = converterFactory.newConverter(stage);
System.out.println("training accuracy = " + accuracy); LogisticRegressionModel lrModel = (LogisticRegressionModel) model.stages()[2]; LogisticRegressionTrainingSummary trainingSummary = lrModel.summary(); double[] objectiveHistory = trainingSummary.objectiveHistory();
numLabels++; int vocabSize = ((CountVectorizerModel)(pipelineModel.stages()[1])).getVocabSize();
CountVectorizerModel cvm = (CountVectorizerModel)pipelineModel.stages()[1]; int vocabSize = cvm.vocabulary().length; numFeatures = Math.min(numFeatures, vocabSize);