@Override public GuaguaWritableAdapter<LongWritable> compute( MasterContext<GuaguaWritableAdapter<LongWritable>, GuaguaWritableAdapter<LongWritable>> context) { long sum = 0l; if(context.getWorkerResults() == null) { LOG.info("Master accumulates worker results with null or empty."); return null; } for(GuaguaWritableAdapter<LongWritable> longWritable: context.getWorkerResults()) { if(longWritable != null) { sum += longWritable.getWritable().get(); } } LOG.info("master:{}", sum); GuaguaWritableAdapter<LongWritable> result = new GuaguaWritableAdapter<LongWritable>(new LongWritable(sum)); if(sum > 1000000L) { result.setHalt(true); } return result; }
@Override public GuaguaWritableAdapter<LongWritable> compute( MasterContext<GuaguaWritableAdapter<LongWritable>, GuaguaWritableAdapter<LongWritable>> context) { long sum = 0l; if(context.getWorkerResults() == null) { LOG.info("Master accumulates worker results with null or empty."); return null; } for(GuaguaWritableAdapter<LongWritable> longWritable: context.getWorkerResults()) { if(longWritable != null) { sum += longWritable.getWritable().get(); } } LOG.info("master:{}", sum); GuaguaWritableAdapter<LongWritable> result = new GuaguaWritableAdapter<LongWritable>(new LongWritable(sum)); if(sum > 1000000L) { result.setHalt(true); } return result; }
/** * Master computation by accumulating all the k center points sum values from all workers, then average to get new k * center points. * * @throws NullPointerException * if worker result or worker results is null. */ @Override public KMeansMasterParams compute(MasterContext<KMeansMasterParams, KMeansWorkerParams> context) { if(context.getWorkerResults() == null) { throw new NullPointerException("No worker results received in Master."); } if(context.getCurrentIteration() == 1) { return doFirstIteration(context); } else { return doOtherIterations(context); } }
/** * Master computation by accumulating all the k center points sum values from all workers, then average to get new k * center points. * * @throws NullPointerException * if worker result or worker results is null. */ @Override public KMeansMasterParams compute(MasterContext<KMeansMasterParams, KMeansWorkerParams> context) { if(context.getWorkerResults() == null) { throw new NullPointerException("No worker results received in Master."); } if(context.getCurrentIteration() == 1) { return doFirstIteration(context); } else { return doOtherIterations(context); } }
/** * Check whether GuaguaConstants.GUAGUA_WORKER_HALT_ENABLE) is enabled, if yes, check whether all workers are halted * and update master status. */ protected void updateMasterHaltStatus(final MasterContext<MASTER_RESULT, WORKER_RESULT> context) { MASTER_RESULT result = context.getMasterResult(); // a switch to make all workers have the right to terminate the application if(Boolean.TRUE.toString().equalsIgnoreCase( context.getProps().getProperty(GuaguaConstants.GUAGUA_WORKER_HALT_ENABLE, GuaguaConstants.GUAGUA_WORKER_DEFAULT_HALT_ENABLE))) { if(isAllWorkersHalt(context.getWorkerResults()) && result instanceof HaltBytable) { ((HaltBytable) result).setHalt(true); context.setMasterResult(result); } } }
@Override public VarSelMasterResult compute(MasterContext<VarSelMasterResult, VarSelWorkerResult> context) { if(this.isInitialized.compareAndSet(false, true)) { init(context); } if(context.getWorkerResults() == null) { throw new IllegalArgumentException("worker's result are null."); } masterConductor.consumeWorkerResults(context.getWorkerResults()); LOG.info("Get results from workers ... "); VarSelMasterResult masterResult = new VarSelMasterResult(masterConductor.getNextWorkingSet()); if(masterConductor.isToStop()) { LOG.info("Variables are selected. Send halt to workers ... "); masterResult.setHalt(true); // save the best seed masterResult.setBestSeed(masterConductor.voteBestSeed()); } else { LOG.info("Send next working set to slaves ... "); } return masterResult; }
for(KMeansWorkerParams workerResult: context.getWorkerResults()) { LOG.debug("Worker result: %s", workerResult); if(!initilized) {
for(KMeansWorkerParams workerResult: context.getWorkerResults()) { LOG.debug("Worker result: %s", workerResult); if(!initilized) {
boolean initilized = false; int k = 0, c = 0; for(KMeansWorkerParams workerResult: context.getWorkerResults()) { allInitialCentriods.addAll(workerResult.getPointList()); if(!initilized) {
boolean initilized = false; int k = 0, c = 0; for(KMeansWorkerParams workerResult: context.getWorkerResults()) { allInitialCentriods.addAll(workerResult.getPointList()); if(!initilized) {
@Override public LogisticRegressionParams compute(MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) { if(context.isFirstIteration()) { init(context); weights = new double[this.inputNum + 1]; for(int i = 0; i < weights.length; i++) { weights[i] = RANDOM.nextDouble(); } } else { double[] gradients = new double[this.inputNum + 1]; double sumError = 0.0d; int size = 0; for(LogisticRegressionParams param: context.getWorkerResults()) { if(param != null) { for(int i = 0; i < gradients.length; i++) { gradients[i] += param.getParameters()[i]; } sumError += param.getError(); } size++; } for(int i = 0; i < weights.length; i++) { weights[i] -= learnRate * gradients[i]; } LOG.debug("DEBUG: Weights: {}", Arrays.toString(this.weights)); LOG.info("Iteration {} with error {}", context.getCurrentIteration(), sumError / size); } return new LogisticRegressionParams(weights); }
@Override public LinearRegressionParams compute(MasterContext<LinearRegressionParams, LinearRegressionParams> context) { if(context.isFirstIteration()) { init(context); weights = new double[this.inputNum + 1]; for(int i = 0; i < weights.length; i++) { weights[i] = RANDOM.nextDouble(); } } else { double[] gradients = new double[this.inputNum + 1]; double sumError = 0.0d; int size = 0; for(LinearRegressionParams param: context.getWorkerResults()) { if(param != null) { for(int i = 0; i < gradients.length; i++) { gradients[i] += param.getParameters()[i]; } sumError += param.getError(); } size++; } for(int i = 0; i < weights.length; i++) { weights[i] -= learnRate * gradients[i]; } LOG.info("DEBUG: Weights: {}", Arrays.toString(this.weights)); LOG.info("Iteration {} with error {}", context.getCurrentIteration(), sumError / size); } return new LinearRegressionParams(weights); }
@Override public LogisticRegressionParams doCompute(MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) { if(context.isFirstIteration()) { initWeights(); } else { double[] gradients = new double[this.inputNum + 1]; double sumError = 0.0d; int size = 0; for(LogisticRegressionParams param: context.getWorkerResults()) { if(param != null) { for(int i = 0; i < gradients.length; i++) { gradients[i] += param.getParameters()[i]; } sumError += param.getError(); } size++; } for(int i = 0; i < weights.length; i++) { weights[i] -= learnRate * gradients[i]; } LOG.debug("DEBUG: Weights: {}", Arrays.toString(this.weights)); LOG.info("Iteration {} with error {}", context.getCurrentIteration(), sumError / size); } return new LogisticRegressionParams(weights); }
@Override public LinearRegressionParams doCompute(MasterContext<LinearRegressionParams, LinearRegressionParams> context) { if(context.isFirstIteration()) { initWeights(); } else { double[] gradients = new double[this.inputNum + 1]; double sumError = 0.0d; int size = 0; for(LinearRegressionParams param: context.getWorkerResults()) { if(param != null) { for(int i = 0; i < gradients.length; i++) { gradients[i] += param.getParameters()[i]; } sumError += param.getError(); } size++; } for(int i = 0; i < weights.length; i++) { weights[i] -= learnRate * gradients[i]; } LOG.info("DEBUG: Weights: {}", Arrays.toString(this.weights)); LOG.info("Iteration {} with error {}", context.getCurrentIteration(), sumError / size); } return new LinearRegressionParams(weights); }
double trainError = 0.0d, testError = 0d; long trainSize = 0, testSize = 0; for(LogisticRegressionParams param: context.getWorkerResults()) { if(param != null) { for(int i = 0; i < gradients.length; i++) {
double trainError = 0d, validationError = 0d; double weightedTrainCount = 0d, weightedValidationCount = 0d; for(DTWorkerParams params: context.getWorkerResults()) { if(!isFirst) { isFirst = true;
if(context.getWorkerResults() == null) { throw new IllegalArgumentException("workers' results are null."); for(NNParams nn: context.getWorkerResults()) { totalTestError += nn.getTestError(); totalTrainError += nn.getTrainError();
if(context.getWorkerResults() == null) { throw new IllegalArgumentException("workers' results are null."); for(NNParams nn: context.getWorkerResults()) { totalTestError += nn.getTestError(); totalTrainError += nn.getTrainError();
if(context.getWorkerResults() == null) { throw new IllegalArgumentException("workers' results are null."); for(NNParams nn: context.getWorkerResults()) { totalTestError += nn.getTestError(); totalTrainError += nn.getTrainError();