private String constructLlapLogUrl(final TezTaskAttemptID attemptID, final String containerIdString, final boolean isDone, final String nmAddress) { String dagId = attemptID.getTaskID().getVertexID().getDAGId().toString(); String filename = JOINER.join(currentHiveQueryId, "-", dagId, ".log", (isDone ? ".done" : ""), "?nm.id=", nmAddress); String url = PATH_JOINER.join(timelineServerUri, "ws", "v1", "applicationhistory", "containers", containerIdString, "logs", filename); return url; }
private static int vertexNum(TaskInfo taskInfo) { return taskInfo.getAttemptId().getTaskID().getVertexID().getId(); // Sigh... }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }
TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600); .setAppAttemptNumber(333).setDagIndex(tezDagId.getId()).build(); builder.setVertexIndex(tezVertexId.getId()); builder.setDagName("dagName"); builder.setVertexName("vertexName");
LOG.info( "Attempting to send terminateRequest for fragment {} due to internal preemption invoked by {}", taskAttemptId.toString(), invokedByContainerEnd ? "containerEnd" : "taskEnd"); LlapNodeId nodeId = entityTracker.getNodeIdForTaskAttempt(taskAttemptId); TerminateFragmentRequestProto.newBuilder().setQueryIdentifier( constructQueryIdentifierProto( taskAttemptId.getTaskID().getVertexID().getDAGId().getId())) .setFragmentIdentifierString(taskAttemptId.toString()).build(); communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(), new LlapProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() { LOG.info( "Not sending terminate request for fragment {} since it's node is not known. Already unregistered", taskAttemptId.toString());
private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId, TaskSpec taskSpec, FragmentRuntimeInfo fragmentRuntimeInfo, String hiveQueryId) throws IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString()); builder.setAmHost(getAmHostString()); builder.setAmPort(getAddress().getPort()); Preconditions.checkState(currentQueryIdentifierProto.getDagIndex() == taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId()); builder.setCredentialsBinary( getCredentials(getContext().getCurrentDagInfo().getCredentials())); builder.setWorkSpec(VertexOrBinary.newBuilder().setVertex(Converters.constructSignableVertexSpec( taskSpec, currentQueryIdentifierProto, getTokenIdentifier(), user, hiveQueryId)).build()); // Don't call builder.setWorkSpecSignature() - Tez doesn't sign fragments builder.setFragmentRuntimeInfo(fragmentRuntimeInfo); if (scheduler != null) { // May be null in tests // TODO: see javadoc builder.setIsGuaranteed(scheduler.isInitialGuaranteed(taskSpec.getTaskAttemptID())); } return builder.build(); }
private SubmitWorkRequestProto createRequest(int fragmentNumber, int numSelfAndUpstreamTasks, int numSelfAndUpstreamComplete, int dagStartTime, int attemptStartTime, int withinDagPriority, String dagName) throws IOException { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return LlapDaemonTestUtils.buildSubmitProtoRequest(fragmentNumber, appId.toString(), dagId.getId(), vId.getId(), dagName, dagStartTime, attemptStartTime, numSelfAndUpstreamTasks, numSelfAndUpstreamComplete, withinDagPriority, new Credentials()); }
taskInfo.attemptId.getTaskID().getVertexID().getDAGId()); Set<Integer> vertexDepInfo = null; if (depInfo != null) {
private void setLlapOfFragmentId(final ProcessorContext context) { // TODO: could we do this only if the OF is actually used? String attemptId = Converters.createTaskAttemptId(context).toString(); if (l4j.isDebugEnabled()) { l4j.debug("Setting the LLAP fragment ID for OF to " + attemptId); } jconf.set(LlapOutputFormat.LLAP_OF_ID_KEY, attemptId); }
TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600); assertEquals("hiveQueryId", vertexProto.getHiveQueryId()); assertEquals(appId.toString(), vertexProto.getQueryIdentifier().getApplicationIdString()); assertEquals(tezDagId.getId(), vertexProto.getQueryIdentifier().getDagIndex()); assertEquals(333, vertexProto.getQueryIdentifier().getAppAttemptNumber()); assertEquals(tezVertexId.getId(), vertexProto.getVertexIndex()); assertEquals(processorDescriptor.getClassName(), vertexProto.getProcessorDescriptor().getClassName());
.setIsGuaranteed(newState).setFragmentIdentifierString(attemptId.toString()) .setQueryIdentifier(constructQueryIdentifierProto( attemptId.getTaskID().getVertexID().getDAGId().getId())).build();
super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials, credentialsChanged, priority); int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId(); if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIndex())) { fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo( taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority); } catch (Exception e) { LOG.error(
public static TezTaskAttemptID createTaskAttemptId(TaskContext ctx) { // Come ride the API roller-coaster #2! The best part is that ctx has TezTaskAttemptID inside. return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ctx.getApplicationId(), ctx.getDagIdentifier()), ctx.getTaskVertexIndex()), ctx.getTaskIndex()), ctx.getTaskAttemptNumber()); }
@Override public void allocateTask(Object task, Resource capability, ContainerId containerId, Priority priority, Object containerSignature, Object clientCookie) { // Container affinity can be implemented as Host affinity for LLAP. Not required until // 1:1 edges are used in Hive. TezTaskAttemptID id = getTaskAttemptId(task); TaskInfo taskInfo = new TaskInfo(localityDelayConf, clock, task, clientCookie, priority, capability, null, null, clock.getTime(), id); LOG.info("Received allocateRequest. task={}, priority={}, capability={}, containerId={}", task, priority, capability, containerId); writeLock.lock(); try { if (!dagRunning && metrics != null && id != null) { metrics.setDagId(id.getTaskID().getVertexID().getDAGId().toString()); } dagRunning = true; dagStats.registerTaskRequest(null, null); } finally { writeLock.unlock(); } addPendingTask(taskInfo); trySchedulingPendingTasks(); }
builder.setQueryIdentifier(queryIdentifierProto); builder.setHiveQueryId(hiveQueryIdString); builder.setVertexIndex(tId.getTaskID().getVertexID().getId()); builder.setDagName(taskSpec.getDAGName()); builder.setVertexName(taskSpec.getVertexName());
private void setLlapOfFragmentId(final ProcessorContext context) { // TODO: could we do this only if the OF is actually used? String attemptId = Converters.createTaskAttemptId(context).toString(); if (l4j.isDebugEnabled()) { l4j.debug("Setting the LLAP fragment ID for OF to " + attemptId); } jconf.set(LlapOutputFormat.LLAP_OF_ID_KEY, attemptId); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
@Override public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks, Priority priority, Object containerSignature, Object clientCookie) { TezTaskAttemptID id = getTaskAttemptId(task); TaskInfo taskInfo = new TaskInfo(localityDelayConf, clock, task, clientCookie, priority, capability, hosts, racks, clock.getTime(), id); LOG.info("Received allocateRequest. task={}, priority={}, capability={}, hosts={}", task, priority, capability, Arrays.toString(hosts)); writeLock.lock(); try { if (!dagRunning && metrics != null && id != null) { metrics.setDagId(id.getTaskID().getVertexID().getDAGId().toString()); } dagRunning = true; dagStats.registerTaskRequest(hosts, racks); } finally { writeLock.unlock(); } addPendingTask(taskInfo); trySchedulingPendingTasks(); }
@Override public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException { String taskAttemptIdString = taskAttemptId.toString(); LlapTaskUmbilicalExternalClient client = registeredClients.get(taskAttemptIdString); if (client != null) { if (client.requestInfo.state == RequestState.PENDING) { // A task kill while the request is still in PENDING state means the request should be retried. LOG.info("Received task kill for {} which is still in pending state. Retry submission.", taskAttemptIdString); client.retrySubmission(); } else { try { LOG.error("Task killed - " + taskAttemptIdString); client.unregisterClient(); if (client.responder != null) { client.responder.taskKilled(taskAttemptId); } } catch (Exception err) { LOG.error("Error during responder execution", err); } } } else { LOG.info("Received task killed notification for task which is not currently being tracked: " + taskAttemptId); } }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }