/** * Whether is in initiate iteration, default initiate iteration is 1. */ public boolean isInitIteration() { return getCurrentIteration() == GuaguaConstants.GUAGUA_INIT_STEP; }
/** * Whether is in first iteration, default first iteration is 1. */ public boolean isFirstIteration() { return getCurrentIteration() == GuaguaConstants.GUAGUA_FIRST_ITERATION; }
@Override public void run() { long start = System.currentTimeMillis(); writeStatesToHdfs(DTMaster.this.checkpointOutput, masterParams, finalTrees, finalIsLeaf, finalTodoQueue, finalToSplitQueue); LOG.info("Do checkpoint in iteration {} with run time {}", context.getCurrentIteration(), (System.currentTimeMillis() - start)); } }, "Master checkpoint thread");
@Override public void preIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { LOG.debug("pre application:{} container:{} iteration:{}, context:{}", context.getAppId(), context.getCurrentIteration(), context); }
@Override public void preIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { LOG.info("pre iteration:{} with memory info {}.", context.getCurrentIteration(), MemoryUtils.getRuntimeMemoryStats()); }
@Override public void postIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { LOG.info("post iteration:{} with memory info {}.", context.getCurrentIteration(), MemoryUtils.getRuntimeMemoryStats()); }
@Override public void preApplication(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { this.coordinator.awaitWorkers(context.getCurrentIteration()); LOG.info("All workers are initilized."); this.coordinator.signalWorkers(context.getCurrentIteration(), null); }
@Override public void postIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { LOG.info("Application {} container {} iteration {} computation ends with {}ms execution time.", context.getAppId(), context.getContainerId(), context.getCurrentIteration(), TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - this.iterStartTime)); }
@Override public void postIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { LOG.info("Application {} container {} iteration {} ends with {}ms execution time.", context.getAppId(), context.getContainerId(), context.getCurrentIteration(), TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - this.iterStartTime)); }
@Override public void preIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { this.coordinator.awaitWorkers(context.getCurrentIteration()); context.setWorkerResults(this.coordinator.getWorkerResults()); LOG.info("All workers are synced in iteration {}.", context.getCurrentIteration()); }
@Override public void preIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { this.iterStartTime = System.nanoTime(); LOG.info("Application {} container {} iteration {} computation starts internal master computatio.", context.getAppId(), context.getContainerId(), context.getCurrentIteration()); }
@Override public void doExecute() throws KeeperException, InterruptedException { String appId = context.getAppId(); int lastIteration = context.getCurrentIteration(); final String appMasterNode = getCurrentMasterNode(appId, lastIteration).toString(); final String appMasterSplitNode = getCurrentMasterSplitNode(appId, lastIteration).toString(); // actually, this is last master result for fault recovery setMasterResult(context, appMasterNode, appMasterSplitNode); } }.execute();
@Override public void preIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { this.iterStartTime = System.nanoTime(); LOG.info("Application {} container {} iteration {} starts master computation.", context.getAppId(), context.getContainerId(), context.getCurrentIteration()); }
@Override public void postIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { LOG.debug("post application:{} container:{} iteration:{}, context:{}", context.getAppId(), context.getContainerId(), context.getCurrentIteration(), context); }
@Override public void postIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { LOG.info("Application {} container {} iteration {} ends with {}ms gc time.", context.getAppId(), context.getContainerId(), context.getCurrentIteration(), TimeUnit.NANOSECONDS.toMillis(computeGCTime() - this.iterGCStartTime)); }
@Override public void postIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { this.coordinator.signalWorkers(context.getCurrentIteration(), context.getMasterResult()); }
@Override public void preIteration(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { this.iterGCStartTime = computeGCTime(); LOG.info("Application {} container {} iteration {} starts master computation.", context.getAppId(), context.getContainerId(), context.getCurrentIteration()); }
/** * Master computation by accumulating all the k center points sum values from all workers, then average to get new k * center points. * * @throws NullPointerException * if worker result or worker results is null. */ @Override public KMeansMasterParams compute(MasterContext<KMeansMasterParams, KMeansWorkerParams> context) { if(context.getWorkerResults() == null) { throw new NullPointerException("No worker results received in Master."); } if(context.getCurrentIteration() == 1) { return doFirstIteration(context); } else { return doOtherIterations(context); } }
/** * Master computation by accumulating all the k center points sum values from all workers, then average to get new k * center points. * * @throws NullPointerException * if worker result or worker results is null. */ @Override public KMeansMasterParams compute(MasterContext<KMeansMasterParams, KMeansWorkerParams> context) { if(context.getWorkerResults() == null) { throw new NullPointerException("No worker results received in Master."); } if(context.getCurrentIteration() == 1) { return doFirstIteration(context); } else { return doOtherIterations(context); } }
protected void setMasterResult(final MasterContext<MASTER_RESULT, WORKER_RESULT> context, final String appMasterNode, final String appMasterSplitNode) throws KeeperException, InterruptedException { if(context.getCurrentIteration() == GuaguaConstants.GUAGUA_INIT_STEP) { return; } byte[] data = getBytesFromZNode(appMasterNode, appMasterSplitNode); if(data != null && data.length > 0) { MASTER_RESULT lastMasterResult = getMasterSerializer().bytesToObject(data, context.getMasterResultClassName()); context.setMasterResult(lastMasterResult); } }