public static TezEvent toTezEvent(NotTezEvent nte) throws InvalidProtocolBufferException { EventMetaData sourceMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), "NULL_VERTEX", null); EventMetaData destMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), nte.getDestInputName(), null); InputDataInformationEvent event = ProtoConverters.convertRootInputDataInformationEventFromProto( RootInputDataInformationEventProto.parseFrom(nte.getInputEventProtoBytes())); TezEvent tezEvent = new TezEvent(event, sourceMetaData, System.currentTimeMillis()); tezEvent.setDestinationInfo(destMetaData); return tezEvent; } }
public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) { IOSpecProto.Builder builder = IOSpecProto.newBuilder(); if (outputSpec.getDestinationVertexName() != null) { builder.setConnectedVertexName(outputSpec.getDestinationVertexName()); } if (outputSpec.getOutputDescriptor() != null) { builder.setIoDescriptor(convertToProto(outputSpec.getOutputDescriptor())); } builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount()); return builder.build(); }
private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) { IOSpecProto.Builder builder = IOSpecProto.newBuilder(); if (inputSpec.getSourceVertexName() != null) { builder.setConnectedVertexName(inputSpec.getSourceVertexName()); } if (inputSpec.getInputDescriptor() != null) { builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor())); } builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount()); return builder.build(); }
TezHeartbeatResponse response = new TezHeartbeatResponse(); response.setLastRequestId(request.getRequestId()); TezTaskAttemptID taskAttemptId = request.getCurrentTaskAttemptID(); String taskAttemptIdString = taskAttemptId.toString(); updateHeartbeatInfo(taskAttemptIdString); response.setShouldDie(); // Do any of the other fields need to be set? return response; response.setLastRequestId(request.getRequestId()); response.setNextFromEventId(0); // Irrelevant. See comment above. response.setNextPreRoutedEventId(0); //Irrelevant. See comment above. response.setEvents(tezEvents); List<TezEvent> inEvents = request.getEvents(); if (LOG.isDebugEnabled()) { LOG.debug("Heartbeat from " + taskAttemptIdString + EventType eventType = tezEvent.getEventType(); switch (eventType) { case TASK_ATTEMPT_COMPLETED_EVENT:
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
@Override public void heartbeat(TezHeartbeatRequest request) { List<TezEvent> inEvents = request.getEvents(); for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) { EventType eventType = tezEvent.getEventType(); try { switch (eventType) { case TASK_ATTEMPT_COMPLETED_EVENT: sendOrQueueEvent(ReaderEvent.doneEvent()); break; case TASK_ATTEMPT_FAILED_EVENT: TaskAttemptFailedEvent taskFailedEvent = (TaskAttemptFailedEvent) tezEvent.getEvent(); sendOrQueueEvent(ReaderEvent.errorEvent(taskFailedEvent.getDiagnostics())); break; case TASK_STATUS_UPDATE_EVENT: // If we want to handle counters break; default: LOG.warn("Unhandled event type " + eventType); break; } } catch (Exception err) { LOG.error("Error during heartbeat responder:", err); } } }
private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
private static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) { GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder(); builder.setGroupName(groupInputSpec.getGroupName()); builder.addAllGroupVertices(groupInputSpec.getGroupVertices()); builder.setMergedInputDescriptor(convertToProto(groupInputSpec.getMergedInputDescriptor())); return builder.build(); }
public void setIsGuaranteed(boolean isGuaranteed) { this.isGuaranteed = isGuaranteed; if (amNodeInfo != null) { amNodeInfo.updateTaskAttempt(taskSpec.getTaskAttemptID(), isGuaranteed); } if (wmCounters != null) { wmCounters.changeGuaranteed(isGuaranteed); } }
private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) { OutputDescriptor outputDescriptor = null; if (outputSpecProto.hasIoDescriptor()) { outputDescriptor = convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor()); } OutputSpec outputSpec = new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor, outputSpecProto.getPhysicalEdgeCount()); return outputSpec; }
private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) { InputDescriptor inputDescriptor = null; if (inputSpecProto.hasIoDescriptor()) { inputDescriptor = convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor()); } InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor, inputSpecProto.getPhysicalEdgeCount()); return inputSpec; }
new TaskSpec(taskAttemptID, vectorProto.getDagName(), vectorProto.getVertexName(), vectorProto.getVertexParallelism(), processorDescriptor, inputSpecList, outputSpecList, groupInputSpecs);
private String extractQueryId(TaskSpec taskSpec) throws IOException { UserPayload processorPayload = taskSpec.getProcessorDescriptor().getUserPayload(); Configuration conf = TezUtils.createConfFromUserPayload(processorPayload); return HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID); }
public HeartbeatCallable( SchedulerFragmentCompletingListener completionListener, RuntimeTask task, LlapTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr, TezEvent initialEvent, String fragmentRequestId, WmFragmentCounters wmCounters) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; this.maxEventsToGet = maxEventsToGet; this.requestCounter = requestCounter; this.containerIdStr = containerIdStr; this.initialEvent = initialEvent; this.completionListener = completionListener; this.fragmentRequestId = fragmentRequestId; this.wmCounters = wmCounters; this.task = task; this.umbilical = umbilical; this.updateEventMetadata = new EventMetaData(EventProducerConsumerType.SYSTEM, task.getVertexName(), "", task.getTaskAttemptID()); nextHeartbeatNumToLog = (Math.max(1, (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f : (float) amPollInterval)))); }
public MockRequest(SubmitWorkRequestProto requestProto, QueryFragmentInfo fragmentInfo, boolean canFinish, boolean canFinishQueue, long workTime, TezEvent initialEvent, boolean isGuaranteed) { super(requestProto, fragmentInfo, new Configuration(), new ExecutionContextImpl("localhost"), null, new Credentials(), 0, mock(AMReporter.class), null, mock( LlapDaemonExecutorMetrics.class), mock(KilledTaskHandler.class), mock( FragmentCompletionHandler.class), new DefaultHadoopShim(), null, requestProto.getWorkSpec().getVertex(), initialEvent, null, mock( SchedulerFragmentCompletingListener.class), mock(SocketFactory.class), isGuaranteed, null); this.workTime = workTime; this.canFinish = canFinish; this.canFinishQueue = canFinishQueue; }
private static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) { GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(), groupInputSpecProto.getGroupVerticesList(), convertInputDescriptorFromProto(groupInputSpecProto.getMergedInputDescriptor())); return groupSpec; }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
private void verifyInputSpecAndProto(InputSpec inputSpec, IOSpecProto inputSpecProto) { assertEquals(inputSpec.getPhysicalEdgeCount(), inputSpecProto.getPhysicalEdgeCount()); assertEquals(inputSpec.getSourceVertexName(), inputSpecProto.getConnectedVertexName()); assertEquals(inputSpec.getInputDescriptor().getClassName(), inputSpecProto.getIoDescriptor().getClassName()); assertEquals(inputSpec.getInputDescriptor().getUserPayload().getPayload(), inputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); }
private void verifyOutputSpecAndProto(OutputSpec outputSpec, IOSpecProto outputSpecProto) { assertEquals(outputSpec.getPhysicalEdgeCount(), outputSpecProto.getPhysicalEdgeCount()); assertEquals(outputSpec.getDestinationVertexName(), outputSpecProto.getConnectedVertexName()); assertEquals(outputSpec.getOutputDescriptor().getClassName(), outputSpecProto.getIoDescriptor().getClassName()); assertEquals(outputSpec.getOutputDescriptor().getUserPayload().getPayload(), outputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); } }
/** * Inform the AM that this task has been killed. */ public void reportTaskKilled() { killedTaskHandler .taskKilled(request.getAmHost(), request.getAmPort(), vertex.getTokenIdentifier(), jobToken, fragmentInfo.getQueryInfo().getQueryIdentifier(), taskSpec.getTaskAttemptID()); }