@Override public void start() { // all services are included in each intecepter. MasterContext<MASTER_RESULT, WORKER_RESULT> context = buildContext(); // iteration 0 is to wait for all workers are available. context.setCurrentIteration(GuaguaConstants.GUAGUA_INIT_STEP); for(MasterInterceptor<MASTER_RESULT, WORKER_RESULT> masterInterceptor: getMasterInterceptors()) { try { masterInterceptor.preApplication(context); } catch (Throwable e) { LOG.error("Error in master interceptors starting.", e); throw new GuaguaRuntimeException(e); } } }
/** * Parameter initialization, such as set result class name, iteration number and so on. */ @Override public void init(Properties props) { this.setProps(props); checkAndSetMasterInterceptors(props); this.setMasterComputable(newMasterComputable()); this.isMonitored = this.getMasterComputable().getClass().isAnnotationPresent(ComputableMonitor.class); if(this.isMonitored) { this.isSoftForComputableTimeout = masterComputable.getClass().getAnnotation(ComputableMonitor.class) .isSoft(); this.executor = Executors.newSingleThreadExecutor(); } this.setTotalIteration(Integer.valueOf(this.getProps().getProperty(GuaguaConstants.GUAGUA_ITERATION_COUNT, Integer.MAX_VALUE + ""))); this.setWorkers(Integer.valueOf(this.getProps().getProperty(GuaguaConstants.GUAGUA_WORKER_NUMBER))); this.setMasterResultClassName(this.getProps().getProperty(GuaguaConstants.GUAGUA_MASTER_RESULT_CLASS)); this.setWorkerResultClassName(this.getProps().getProperty(GuaguaConstants.GUAGUA_WORKER_RESULT_CLASS)); double minWorkersRatio = NumberFormatUtils.getDouble( this.getProps().getProperty(GuaguaConstants.GUAGUA_MIN_WORKERS_RATIO), GuaguaConstants.GUAGUA_DEFAULT_MIN_WORKERS_RATIO); this.setMinWorkersRatio(minWorkersRatio); long minWorkersTimeOut = NumberFormatUtils.getLong( this.getProps().getProperty(GuaguaConstants.GUAGUA_MIN_WORKERS_TIMEOUT), 5 * 1000l); this.setMinWorkersTimeOut(minWorkersTimeOut); }
private MasterContext<MASTER_RESULT, WORKER_RESULT> buildContext() { if(getContext() != null) { return getContext(); } this.context = new MasterContext<MASTER_RESULT, WORKER_RESULT>(getTotalIteration(), getWorkers(), getProps(), getAppId(), getContainerId(), getMasterResultClassName(), getWorkerResultClassName(), getMinWorkersRatio(), getMinWorkersTimeOut()); return getContext(); }
@Override protected void setup(Context context) throws java.io.IOException, InterruptedException { GuaguaInputSplit inputSplit = (GuaguaInputSplit) context.getInputSplit(); this.setMaster(inputSplit.isMaster()); if(this.isMaster()) { context.setStatus("Master initializing ..."); this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { context.setStatus("Worker initializing ..."); this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(int i = 0; i < inputSplit.getFileSplits().length; i++) { FileSplit fs = inputSplit.getFileSplits()[i]; GuaguaFileSplit gfs = new GuaguaFileSplit(fs.getPath().toString(), fs.getStart(), fs.getLength()); if(inputSplit.getExtensions() != null && i < inputSplit.getExtensions().length) { gfs.setExtension(inputSplit.getExtensions()[i]); } splits.add(gfs); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(context.getConfiguration()); this.getGuaguaService().setAppId(context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_JOB_ID)); this.getGuaguaService().setContainerId( context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_TASK_PARTITION)); this.getGuaguaService().init(props); this.getGuaguaService().start(); }
this.masterService = new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>(); this.masterService.setAppId(GUAGUA_UNIT_TEST); this.masterService.setContainerId("0"); ((GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>) this.masterService).setCoordinator(coordinator); this.masterService.init(this.props);
MasterContext<MASTER_RESULT, WORKER_RESULT> context = buildContext(); int initialIteration = context.getCurrentIteration(); try { for(int i = initialIteration; i < getTotalIteration(); i++) { int iteration = i + 1; context.setCurrentIteration(iteration); iterate(context, iteration, progress);
String status = "Start master iteration ( %s/%s ), progress %s%%"; if(progress != null) { progress.progress(iteration - 1, getTotalIteration(), String.format(status, iteration, getTotalIteration(), ((iteration - 1) * 100 / getTotalIteration())), false, false); for(MasterInterceptor<MASTER_RESULT, WORKER_RESULT> masterInterceptor: getMasterInterceptors()) { masterInterceptor.preIteration(context); progress.progress(iteration - 1, getTotalIteration(), String.format(status, iteration, getTotalIteration(), ((iteration - 1) * 100 / getTotalIteration())), false, false); progress.progress(iteration - 1, getTotalIteration(), String.format(status, iteration, getTotalIteration(), ((iteration - 1) * 100 / getTotalIteration())), false, false); int interceptorsSize = getMasterInterceptors().size(); for(int i = 0; i < interceptorsSize; i++) { getMasterInterceptors().get(interceptorsSize - 1 - i).postIteration(context); progress.progress(iteration, getTotalIteration(), String.format(status, iteration, getTotalIteration(), (iteration * 100 / getTotalIteration())), true, false);
public void addMasterInterceptors(MasterInterceptor<MASTER_RESULT, WORKER_RESULT> masterInterceptor) { getMasterInterceptors().add(masterInterceptor); }
@Override protected void setup(Context context) throws java.io.IOException, InterruptedException { GuaguaInputSplit inputSplit = (GuaguaInputSplit) context.getInputSplit(); this.setMaster(inputSplit.isMaster()); if(this.isMaster()) { context.setStatus("Master initializing ..."); this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { context.setStatus("Worker initializing ..."); this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(int i = 0; i < inputSplit.getFileSplits().length; i++) { FileSplit fs = inputSplit.getFileSplits()[i]; GuaguaFileSplit gfs = new GuaguaFileSplit(fs.getPath().toString(), fs.getStart(), fs.getLength()); if(inputSplit.getExtensions() != null && i < inputSplit.getExtensions().length) { gfs.setExtension(inputSplit.getExtensions()[i]); } splits.add(gfs); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(context.getConfiguration()); this.getGuaguaService().setAppId(context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_JOB_ID)); this.getGuaguaService().setContainerId( context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_TASK_PARTITION)); this.getGuaguaService().init(props); this.getGuaguaService().start(); }
/** * Set up guagua service */ protected void setup() { this.setMaster(this.getInputSplit().isMaster()); if(this.isMaster()) { this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(FileSplit fileSplit: getInputSplit().getFileSplits()) { splits.add(new GuaguaFileSplit(fileSplit.getPath().toString(), fileSplit.getStart(), fileSplit .getLength())); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(); this.getGuaguaService().setAppId(this.getAppId().toString()); this.getGuaguaService().setContainerId(this.getPartition() + ""); this.getGuaguaService().init(props); this.getGuaguaService().start(); initRPCClient(); }
MasterContext<MASTER_RESULT, WORKER_RESULT> context = buildContext(); int interceptorsSize = getMasterInterceptors().size(); Throwable exception = null; for(int i = 0; i < interceptorsSize; i++) { try { getMasterInterceptors().get(interceptorsSize - 1 - i).postApplication(context); } catch (Throwable e) {
/** * Set up guagua service */ protected void setup() { this.setMaster(this.getInputSplit().isMaster()); if(this.isMaster()) { this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(FileSplit fileSplit: getInputSplit().getFileSplits()) { splits.add(new GuaguaFileSplit(fileSplit.getPath().toString(), fileSplit.getStart(), fileSplit .getLength())); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(); this.getGuaguaService().setAppId(this.getAppId().toString()); this.getGuaguaService().setContainerId(this.getPartition() + ""); this.getGuaguaService().init(props); this.getGuaguaService().start(); initRPCClient(); }