/** * @param ctx Kernal context. */ public HadoopContext( GridKernalContext ctx, HadoopConfiguration cfg, HadoopJobTracker jobTracker, HadoopTaskExecutorAdapter taskExecutor, HadoopShuffle shuffle ) { this.ctx = ctx; this.cfg = cfg; this.jobTracker = add(jobTracker); this.taskExecutor = add(taskExecutor); this.shuffle = add(shuffle); }
/** {@inheritDoc} */ @Override public void onKernalStop(boolean cancel) { super.onKernalStop(cancel); if (hctx == null) return; List<HadoopComponent> components = hctx.components(); for (ListIterator<HadoopComponent> it = components.listIterator(components.size()); it.hasPrevious();) { HadoopComponent c = it.previous(); c.onKernalStop(cancel); } }
/** {@inheritDoc} */ @Override public void onKernalStart() throws IgniteCheckedException { super.onKernalStart(); jobTracker = ctx.jobTracker(); exec = new HadoopExecutorService(log, ctx.kernalContext().igniteInstanceName(), ctx.configuration().getMaxParallelTasks(), ctx.configuration().getMaxTaskQueueSize()); }
/** * @param meta Job metadata. * @return {@code true} If local node is participating in job execution. */ public boolean isParticipating(HadoopJobMetadata meta) { UUID locNodeId = localNodeId(); if (locNodeId.equals(meta.submitNodeId())) return true; HadoopMapReducePlan plan = meta.mapReducePlan(); return plan.mapperNodeIds().contains(locNodeId) || plan.reducerNodeIds().contains(locNodeId) || jobUpdateLeader(); }
/** * @return {@code True} if */ public boolean jobUpdateLeader() { long minOrder = Long.MAX_VALUE; ClusterNode minOrderNode = null; for (ClusterNode node : nodes()) { if (node.order() < minOrder) { minOrder = node.order(); minOrderNode = node; } } assert minOrderNode != null; return localNodeId().equals(minOrderNode.id()); }
private void processNodeLeft(DiscoveryEvent evt) { if (log.isDebugEnabled()) log.debug("Processing discovery event [locNodeId=" + ctx.localNodeId() + ", evt=" + evt + ']'); if (ctx.jobUpdateLeader()) { boolean checkSetup = evt.eventNode().order() < ctx.localNodeOrder(); ctx.taskExecutor().run(job, setupTask); if (ctx.kernalContext().discovery().node(nodeId) == null) { if (ctx.kernalContext().discovery().node(nodeId) == null) {
if (ctx.jobUpdateLeader()) { Collection<HadoopTaskInfo> setupTask = setupTask(jobId); ctx.taskExecutor().run(job, setupTask); ctx.taskExecutor().run(job, tasks); if (meta.pendingReducers().isEmpty() && ctx.jobUpdateLeader()) { HadoopTaskInfo info = new HadoopTaskInfo(COMMIT, jobId, 0, 0, null); ctx.taskExecutor().run(job, Collections.singletonList(info)); ctx.taskExecutor().run(job, tasks); ctx.taskExecutor().cancelTasks(jobId); if (ctx.jobUpdateLeader()) { if (state == null) state = initState(jobId); ctx.taskExecutor().run(job, Collections.singletonList(info)); Collection<HadoopInputSplit> mappers = plan.mappers(ctx.localNodeId()); int[] rdc = plan.reducers(ctx.localNodeId()); ctx.shuffle().jobFinished(jobId); if (ctx.jobUpdateLeader())
/** {@inheritDoc} */ @Override public void start(HadoopContext ctx) throws IgniteCheckedException { this.ctx = ctx; log = ctx.kernalContext().log(HadoopExternalTaskExecutor.class); outputBase = U.resolveWorkDirectory(ctx.kernalContext().config().getWorkDirectory(), "hadoop", false); pathSep = System.getProperty("path.separator", U.isWindows() ? ";" : ":"); initJavaCommand(); comm = new HadoopExternalCommunication( ctx.localNodeId(), UUID.randomUUID(), ctx.kernalContext().config().getMarshaller(), log, ctx.kernalContext().getSystemExecutorService(), ctx.kernalContext().igniteInstanceName(), ctx.kernalContext().config().getWorkDirectory()); comm.setListener(new MessageListener()); comm.start(); nodeDesc = comm.localProcessDescriptor(); ctx.kernalContext().ports().registerPort(nodeDesc.tcpPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class); if (nodeDesc.sharedMemoryPort() != -1) ctx.kernalContext().ports().registerPort(nodeDesc.sharedMemoryPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class); jobTracker = ctx.jobTracker(); }
synchronized (mux) { if ((prj = jobMetaPrj) == null) { GridCacheAdapter<HadoopJobId, HadoopJobMetadata> sysCache = ctx.kernalContext().cache() .internalCache(CU.SYS_CACHE_HADOOP_MR); mrPlanner = ctx.planner(); ctx.kernalContext().resource().injectGeneric(mrPlanner); if (ctx.configuration().getFinishedJobInfoTtl() > 0) { ExpiryPolicy finishedJobPlc = new ModifiedExpiryPolicy( new Duration(MILLISECONDS, ctx.configuration().getFinishedJobInfoTtl()));
/** * @param updated Updated cache entries. * @throws IgniteCheckedException If failed. */ private void processJobMetadataUpdates( Iterable<CacheEntryEvent<? extends HadoopJobId, ? extends HadoopJobMetadata>> updated) throws IgniteCheckedException { UUID locNodeId = ctx.localNodeId(); for (CacheEntryEvent<? extends HadoopJobId, ? extends HadoopJobMetadata> entry : updated) { HadoopJobId jobId = entry.getKey(); HadoopJobMetadata meta = entry.getValue(); if (meta == null || !ctx.isParticipating(meta)) continue; if (log.isDebugEnabled()) log.debug("Processing job metadata update callback [locNodeId=" + locNodeId + ", meta=" + meta + ']'); try { ctx.taskExecutor().onJobStateChanged(meta); } catch (IgniteCheckedException e) { U.error(log, "Failed to process job state changed callback (will fail the job) " + "[locNodeId=" + locNodeId + ", jobId=" + jobId + ", meta=" + meta + ']', e); transform(jobId, new CancelJobProcessor(null, e)); continue; } processJobMetaUpdate(jobId, meta, locNodeId); } }
job = jobInfo.createJob(jobCls, jobId, log, ctx.configuration().getNativeLibraryNames(), ctx.kernalContext().hadoopHelper()); job.initialize(false, ctx.localNodeId());
/** * @param ctx Hadoop context. */ public void start(HadoopContext ctx) throws IgniteCheckedException { this.ctx = ctx; log = ctx.kernalContext().log(getClass()); }
/** {@inheritDoc} */ @Override public HadoopCounters counters(HadoopJobId jobId) throws IgniteCheckedException { return hctx.jobTracker().jobCounters(jobId); }
/** * Get number of local mappers. * * @param plan Plan. * @return Number of local mappers. */ private int localMappersCount(HadoopMapReducePlan plan) { Collection<HadoopInputSplit> locMappers = plan.mappers(ctx.localNodeId()); return F.isEmpty(locMappers) ? 0 : locMappers.size(); }
/** * Creates new shuffle job. * * @param jobId Job ID. * @return Created shuffle job. * @throws IgniteCheckedException If job creation failed. */ private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedException { HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId); HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log, ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()), localMappersCount(plan), true); UUID[] rdcAddrs = new UUID[plan.reducers()]; for (int i = 0; i < rdcAddrs.length; i++) { UUID nodeId = plan.nodeForReducer(i); assert nodeId != null : "Plan is missing node for reducer [plan=" + plan + ", rdc=" + i + ']'; rdcAddrs[i] = nodeId; } boolean init = job.initializeReduceAddresses(rdcAddrs); assert init; return job; }
log.debug("Submitting tasks for local execution [locNodeId=" + ctx.localNodeId() + ", tasksCnt=" + tasks.size() + ']'); assert info != null; HadoopRunnableTask task = new HadoopRunnableTask(log, job, ctx.shuffle().memory(), info, ctx.localNodeId()) { @Override protected void onTaskFinished(HadoopTaskStatus status) { if (log.isDebugEnabled())
final HadoopProcess proc = new HadoopProcess(jobId, fut, plan.reducers(ctx.localNodeId())); ctx.kernalContext().closure().runLocalSafe(new Runnable() { @Override public void run() { if (!busyLock.tryReadLock()) {
/** {@inheritDoc} */ @Override public void start() throws IgniteCheckedException { if (ctx.isDaemon()) return; HadoopConfiguration cfg = ctx.config().getHadoopConfiguration(); if (cfg == null) cfg = new HadoopConfiguration(); else cfg = new HadoopConfiguration(cfg); initializeDefaults(cfg); hctx = new HadoopContext( ctx, cfg, new HadoopJobTracker(), new HadoopEmbeddedTaskExecutor(), // TODO: IGNITE-404: Uncomment when fixed. //cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(), new HadoopShuffle()); for (HadoopComponent c : hctx.components()) c.start(hctx); hadoop = new HadoopImpl(this); ctx.addNodeAttribute(HadoopAttributes.NAME, new HadoopAttributes(cfg)); }
/** {@inheritDoc} */ @Override public HadoopConfiguration config() { return hctx.configuration(); }
else if (ctx.isParticipating(meta)) { HadoopJobEx job;