/** {@inheritDoc} */ @Override public HadoopCounters counters(HadoopJobId jobId) throws IgniteCheckedException { return hctx.jobTracker().jobCounters(jobId); }
/** {@inheritDoc} */ @Override public IgniteInternalFuture<?> finishFuture(HadoopJobId jobId) throws IgniteCheckedException { return hctx.jobTracker().finishFuture(jobId); }
/** {@inheritDoc} */ @Override public HadoopJobStatus status(HadoopJobId jobId) throws IgniteCheckedException { return hctx.jobTracker().status(jobId); }
/** {@inheritDoc} */ @Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException { return hctx.jobTracker().killJob(jobId); }
/** {@inheritDoc} */ @Override public IgniteInternalFuture<?> submit(HadoopJobId jobId, HadoopJobInfo jobInfo) { ClassLoader oldLdr = HadoopCommonUtils.setContextClassLoader(getClass().getClassLoader()); try { return hctx.jobTracker().submit(jobId, jobInfo); } finally { HadoopCommonUtils.restoreContextClassLoader(oldLdr); } }
/** * Creates new shuffle job. * * @param jobId Job ID. * @return Created shuffle job. * @throws IgniteCheckedException If job creation failed. */ private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedException { HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId); HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log, ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()), localMappersCount(plan), true); UUID[] rdcAddrs = new UUID[plan.reducers()]; for (int i = 0; i < rdcAddrs.length; i++) { UUID nodeId = plan.nodeForReducer(i); assert nodeId != null : "Plan is missing node for reducer [plan=" + plan + ", rdc=" + i + ']'; rdcAddrs[i] = nodeId; } boolean init = job.initializeReduceAddresses(rdcAddrs); assert init; return job; }
/** {@inheritDoc} */ @Override public void onKernalStart() throws IgniteCheckedException { super.onKernalStart(); jobTracker = ctx.jobTracker(); exec = new HadoopExecutorService(log, ctx.kernalContext().igniteInstanceName(), ctx.configuration().getMaxParallelTasks(), ctx.configuration().getMaxTaskQueueSize()); }
/** {@inheritDoc} */ @Override public void start(HadoopContext ctx) throws IgniteCheckedException { this.ctx = ctx; log = ctx.kernalContext().log(HadoopExternalTaskExecutor.class); outputBase = U.resolveWorkDirectory(ctx.kernalContext().config().getWorkDirectory(), "hadoop", false); pathSep = System.getProperty("path.separator", U.isWindows() ? ";" : ":"); initJavaCommand(); comm = new HadoopExternalCommunication( ctx.localNodeId(), UUID.randomUUID(), ctx.kernalContext().config().getMarshaller(), log, ctx.kernalContext().getSystemExecutorService(), ctx.kernalContext().igniteInstanceName(), ctx.kernalContext().config().getWorkDirectory()); comm.setListener(new MessageListener()); comm.start(); nodeDesc = comm.localProcessDescriptor(); ctx.kernalContext().ports().registerPort(nodeDesc.tcpPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class); if (nodeDesc.sharedMemoryPort() != -1) ctx.kernalContext().ports().registerPort(nodeDesc.sharedMemoryPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class); jobTracker = ctx.jobTracker(); }