this.masterService.setAppId(GUAGUA_UNIT_TEST); this.masterService.setContainerId("0"); ((GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>) this.masterService).setCoordinator(coordinator); this.masterService.init(this.props); workerService.setAppId(GUAGUA_UNIT_TEST); workerService.setContainerId((i + 1) + ""); workerService.setSplits(Arrays.asList(this.fileSplits.get(i))); ((GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>) workerService).setCoordinator(coordinator); workerService.init(this.props); this.workerServices.add(workerService);
@Override protected void cleanup(org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, Text, Text>.Context context) throws IOException, InterruptedException { this.getGuaguaService().stop(); }
try { this.setup(); this.getGuaguaService().run(new Progressable() { @Override public void progress(int currentIteration, int totalIteration, String status, boolean isLastUpdate,
@Override protected void setup(Context context) throws java.io.IOException, InterruptedException { GuaguaInputSplit inputSplit = (GuaguaInputSplit) context.getInputSplit(); this.setMaster(inputSplit.isMaster()); if(this.isMaster()) { context.setStatus("Master initializing ..."); this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { context.setStatus("Worker initializing ..."); this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(int i = 0; i < inputSplit.getFileSplits().length; i++) { FileSplit fs = inputSplit.getFileSplits()[i]; GuaguaFileSplit gfs = new GuaguaFileSplit(fs.getPath().toString(), fs.getStart(), fs.getLength()); if(inputSplit.getExtensions() != null && i < inputSplit.getExtensions().length) { gfs.setExtension(inputSplit.getExtensions()[i]); } splits.add(gfs); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(context.getConfiguration()); this.getGuaguaService().setAppId(context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_JOB_ID)); this.getGuaguaService().setContainerId( context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_TASK_PARTITION)); this.getGuaguaService().init(props); this.getGuaguaService().start(); }
@Override protected void cleanup(org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, Text, Text>.Context context) throws IOException, InterruptedException { this.getGuaguaService().stop(); }
try { this.setup(); this.getGuaguaService().run(new Progressable() { @Override public void progress(int currentIteration, int totalIteration, String status, boolean isLastUpdate,
@Override protected void setup(Context context) throws java.io.IOException, InterruptedException { GuaguaInputSplit inputSplit = (GuaguaInputSplit) context.getInputSplit(); this.setMaster(inputSplit.isMaster()); if(this.isMaster()) { context.setStatus("Master initializing ..."); this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { context.setStatus("Worker initializing ..."); this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(int i = 0; i < inputSplit.getFileSplits().length; i++) { FileSplit fs = inputSplit.getFileSplits()[i]; GuaguaFileSplit gfs = new GuaguaFileSplit(fs.getPath().toString(), fs.getStart(), fs.getLength()); if(inputSplit.getExtensions() != null && i < inputSplit.getExtensions().length) { gfs.setExtension(inputSplit.getExtensions()[i]); } splits.add(gfs); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(context.getConfiguration()); this.getGuaguaService().setAppId(context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_JOB_ID)); this.getGuaguaService().setContainerId( context.getConfiguration().get(GuaguaMapReduceConstants.MAPRED_TASK_PARTITION)); this.getGuaguaService().init(props); this.getGuaguaService().start(); }
/** * Clean up resources used */ protected void cleanup() { if(this.rpcClient != null) { this.rpcClient.shutdown(); this.rpcClient.releaseExternalResources(); } if(this.rpcClientChannel != null) { this.rpcClientChannel.close(); } this.getGuaguaService().stop(); }
this.setup(context); final int iterations = context.getConfiguration().getInt(GuaguaConstants.GUAGUA_ITERATION_COUNT, -1); this.getGuaguaService().run(new Progressable() {
/** * Set up guagua service */ protected void setup() { this.setMaster(this.getInputSplit().isMaster()); if(this.isMaster()) { this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(FileSplit fileSplit: getInputSplit().getFileSplits()) { splits.add(new GuaguaFileSplit(fileSplit.getPath().toString(), fileSplit.getStart(), fileSplit .getLength())); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(); this.getGuaguaService().setAppId(this.getAppId().toString()); this.getGuaguaService().setContainerId(this.getPartition() + ""); this.getGuaguaService().init(props); this.getGuaguaService().start(); initRPCClient(); }
/** * Clean up resources used */ protected void cleanup() { if(this.rpcClient != null) { this.rpcClient.shutdown(); this.rpcClient.releaseExternalResources(); } if(this.rpcClientChannel != null) { this.rpcClientChannel.close(); } this.getGuaguaService().stop(); }
this.setup(context); final int iterations = context.getConfiguration().getInt(GuaguaConstants.GUAGUA_ITERATION_COUNT, -1); this.getGuaguaService().run(new Progressable() {
/** * Set up guagua service */ protected void setup() { this.setMaster(this.getInputSplit().isMaster()); if(this.isMaster()) { this.setGuaguaService(new GuaguaMasterService<MASTER_RESULT, WORKER_RESULT>()); } else { this.setGuaguaService(new GuaguaWorkerService<MASTER_RESULT, WORKER_RESULT>()); List<GuaguaFileSplit> splits = new LinkedList<GuaguaFileSplit>(); for(FileSplit fileSplit: getInputSplit().getFileSplits()) { splits.add(new GuaguaFileSplit(fileSplit.getPath().toString(), fileSplit.getStart(), fileSplit .getLength())); } this.getGuaguaService().setSplits(splits); } Properties props = replaceConfToProps(); this.getGuaguaService().setAppId(this.getAppId().toString()); this.getGuaguaService().setContainerId(this.getPartition() + ""); this.getGuaguaService().init(props); this.getGuaguaService().start(); initRPCClient(); }