/** * Get number of local mappers. * * @param plan Plan. * @return Number of local mappers. */ private int localMappersCount(HadoopMapReducePlan plan) { Collection<HadoopInputSplit> locMappers = plan.mappers(ctx.localNodeId()); return F.isEmpty(locMappers) ? 0 : locMappers.size(); }
/** * @param meta Job metadata. * @return {@code true} If local node is participating in job execution. */ public boolean isParticipating(HadoopJobMetadata meta) { UUID locNodeId = localNodeId(); if (locNodeId.equals(meta.submitNodeId())) return true; HadoopMapReducePlan plan = meta.mapReducePlan(); return plan.mapperNodeIds().contains(locNodeId) || plan.reducerNodeIds().contains(locNodeId) || jobUpdateLeader(); }
/** * @return {@code True} if */ public boolean jobUpdateLeader() { long minOrder = Long.MAX_VALUE; ClusterNode minOrderNode = null; for (ClusterNode node : nodes()) { if (node.order() < minOrder) { minOrder = node.order(); minOrderNode = node; } } assert minOrderNode != null; return localNodeId().equals(minOrderNode.id()); }
log.debug("Submitting tasks for local execution [locNodeId=" + ctx.localNodeId() + ", tasksCnt=" + tasks.size() + ']'); ctx.localNodeId()) { @Override protected void onTaskFinished(HadoopTaskStatus status) { if (log.isDebugEnabled())
/** * Creates reducer tasks based on job information. * * @param reducers Reducers (may be {@code null}). * @param job Job instance. * @return Collection of task infos. */ private Collection<HadoopTaskInfo> reducerTasks(int[] reducers, HadoopJobEx job) { UUID locNodeId = ctx.localNodeId(); HadoopJobId jobId = job.id(); JobLocalState state = activeJobs.get(jobId); Collection<HadoopTaskInfo> tasks = null; if (reducers != null) { if (state == null) state = initState(job.id()); for (int rdc : reducers) { if (state.addReducer(rdc)) { if (log.isDebugEnabled()) log.debug("Submitting REDUCE task for execution [locNodeId=" + locNodeId + ", rdc=" + rdc + ']'); HadoopTaskInfo taskInfo = new HadoopTaskInfo(REDUCE, jobId, rdc, 0, null); if (tasks == null) tasks = new ArrayList<>(); tasks.add(taskInfo); } } } return tasks; }
UUID locNodeId = ctx.localNodeId(); HadoopJobId jobId = meta.jobId();
final HadoopProcess proc = new HadoopProcess(jobId, fut, plan.reducers(ctx.localNodeId()));
/** * Creates new shuffle job. * * @param jobId Job ID. * @return Created shuffle job. * @throws IgniteCheckedException If job creation failed. */ private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedException { HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId); HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log, ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()), localMappersCount(plan), true); UUID[] rdcAddrs = new UUID[plan.reducers()]; for (int i = 0; i < rdcAddrs.length; i++) { UUID nodeId = plan.nodeForReducer(i); assert nodeId != null : "Plan is missing node for reducer [plan=" + plan + ", rdc=" + i + ']'; rdcAddrs[i] = nodeId; } boolean init = job.initializeReduceAddresses(rdcAddrs); assert init; return job; }
ctx.kernalContext().hadoopHelper()); job.initialize(false, ctx.localNodeId());
/** * Sends prepare request to remote process. * * @param proc Process to send request to. * @param job Job. * @param plan Map reduce plan. */ private void prepareForJob(HadoopProcess proc, HadoopJobEx job, HadoopMapReducePlan plan) { try { comm.sendMessage(proc.descriptor(), new HadoopPrepareForJobRequest(job.id(), job.info(), plan.reducers(), plan.reducers(ctx.localNodeId()))); } catch (IgniteCheckedException e) { U.error(log, "Failed to send job prepare request to remote process [proc=" + proc + ", job=" + job + ", plan=" + plan + ']', e); proc.terminate(); } }
log.trace("Got job metadata for status check [locNodeId=" + ctx.localNodeId() + ", meta=" + meta + ']'); log.trace("Re-checking job metadata [locNodeId=" + ctx.localNodeId() + ", meta=" + meta + ']');
Collection<Integer> cancelReducers = new ArrayList<>(); Collection<HadoopInputSplit> mappers = plan.mappers(ctx.localNodeId()); int[] rdc = plan.reducers(ctx.localNodeId());
/** * @param updated Updated cache entries. * @throws IgniteCheckedException If failed. */ private void processJobMetadataUpdates( Iterable<CacheEntryEvent<? extends HadoopJobId, ? extends HadoopJobMetadata>> updated) throws IgniteCheckedException { UUID locNodeId = ctx.localNodeId(); for (CacheEntryEvent<? extends HadoopJobId, ? extends HadoopJobMetadata> entry : updated) { HadoopJobId jobId = entry.getKey(); HadoopJobMetadata meta = entry.getValue(); if (meta == null || !ctx.isParticipating(meta)) continue; if (log.isDebugEnabled()) log.debug("Processing job metadata update callback [locNodeId=" + locNodeId + ", meta=" + meta + ']'); try { ctx.taskExecutor().onJobStateChanged(meta); } catch (IgniteCheckedException e) { U.error(log, "Failed to process job state changed callback (will fail the job) " + "[locNodeId=" + locNodeId + ", jobId=" + jobId + ", meta=" + meta + ']', e); transform(jobId, new CancelJobProcessor(null, e)); continue; } processJobMetaUpdate(jobId, meta, locNodeId); } }
HadoopJobMetadata meta = new HadoopJobMetadata(ctx.localNodeId(), jobId, info); ctx.localNodeId());
private void processNodeLeft(DiscoveryEvent evt) { if (log.isDebugEnabled()) log.debug("Processing discovery event [locNodeId=" + ctx.localNodeId() + ", evt=" + evt + ']');
/** {@inheritDoc} */ @Override public void start(HadoopContext ctx) throws IgniteCheckedException { this.ctx = ctx; log = ctx.kernalContext().log(HadoopExternalTaskExecutor.class); outputBase = U.resolveWorkDirectory(ctx.kernalContext().config().getWorkDirectory(), "hadoop", false); pathSep = System.getProperty("path.separator", U.isWindows() ? ";" : ":"); initJavaCommand(); comm = new HadoopExternalCommunication( ctx.localNodeId(), UUID.randomUUID(), ctx.kernalContext().config().getMarshaller(), log, ctx.kernalContext().getSystemExecutorService(), ctx.kernalContext().igniteInstanceName(), ctx.kernalContext().config().getWorkDirectory()); comm.setListener(new MessageListener()); comm.start(); nodeDesc = comm.localProcessDescriptor(); ctx.kernalContext().ports().registerPort(nodeDesc.tcpPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class); if (nodeDesc.sharedMemoryPort() != -1) ctx.kernalContext().ports().registerPort(nodeDesc.sharedMemoryPort(), IgnitePortProtocol.TCP, HadoopExternalTaskExecutor.class); jobTracker = ctx.jobTracker(); }