fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo( taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority); } catch (Exception e) { LOG.error(
private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId, TaskSpec taskSpec, FragmentRuntimeInfo fragmentRuntimeInfo, String hiveQueryId) throws IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString()); builder.setAmHost(getAmHostString()); builder.setAmPort(getAddress().getPort()); Preconditions.checkState(currentQueryIdentifierProto.getDagIndex() == taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId()); builder.setCredentialsBinary( getCredentials(getContext().getCurrentDagInfo().getCredentials())); builder.setWorkSpec(VertexOrBinary.newBuilder().setVertex(Converters.constructSignableVertexSpec( taskSpec, currentQueryIdentifierProto, getTokenIdentifier(), user, hiveQueryId)).build()); // Don't call builder.setWorkSpecSignature() - Tez doesn't sign fragments builder.setFragmentRuntimeInfo(fragmentRuntimeInfo); if (scheduler != null) { // May be null in tests // TODO: see javadoc builder.setIsGuaranteed(scheduler.isInitialGuaranteed(taskSpec.getTaskAttemptID())); } return builder.build(); }
@Override public int getIdentifier() { return taskId.getId(); }
boolean containsTask(TezTaskID taskID) { return taskSet.get(taskID.getId()); } }
void addTask(TezTaskID taskID) { taskSet.set(taskID.getId()); }
private void taskAttemptSeen(String vertexName, TezTaskAttemptID taskAttemptID) { BitSet scheduledTasks = vertexScheduledTasks.get(vertexName); if (scheduledTasks == null) { scheduledTasks = new BitSet(); vertexScheduledTasks.put(vertexName, scheduledTasks); } if (taskAttemptID != null) { // null for 0 task vertices scheduledTasks.set(taskAttemptID.getTaskID().getId()); } }
@Override public int getTaskIndex() { return taskAttemptID.getTaskID().getId(); }
@Override public void scheduleSpeculativeTask(TezTaskID taskId) { readLock.lock(); try { Preconditions.checkState(taskId.getId() < numTasks); eventHandler.handle(new TaskEvent(taskId, TaskEventType.T_ADD_SPEC_ATTEMPT)); } finally { readLock.unlock(); } }
Integer successfulAttempt = vertexSuccessfulAttemptMap.get(taskId.getId()); if (successfulAttempt == null) { successfulAttempt = attemptId; vertexSuccessfulAttemptMap.put(taskId.getId(), successfulAttempt); while (eventIterator.hasNext()) { TezEvent tezEvent = eventIterator.next(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId(); if (taskIndex == taskId.getId()) {
public static org.apache.hadoop.mapred.TaskID createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getVertexID(); ApplicationId appId = vId.getDAGId().getApplicationId(); return new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp()) + String.valueOf(vId.getId()), appId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getId()); }
public static TaskID toMRTaskId(TezTaskID taskid) { return new TaskID( toMRJobId(taskid.getVertexID().getDAGId()), taskid.getVertexID().getId() == 0 ? TaskType.MAP : TaskType.REDUCE, taskid.getId()); }
committer.recoverTask(task.getTaskId().getId(), task.appContext.getApplicationAttemptId().getAttemptId()-1); } catch (Exception e) {
public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getTaskID().getVertexID(); ApplicationId appId = vId.getDAGId().getApplicationId(); return new org.apache.hadoop.mapred.TaskAttemptID( new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp()) + String.valueOf(vId.getId()), appId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getTaskID().getId()), tezTaId.getId()); }
public List<TezEvent> generateEmptyEventsForAttempt(TezTaskAttemptID attempt) throws Exception { if (!edgeProperty.getEdgeSource().getClassName().startsWith("org.apache.tez")) { throw new TezException("Only org.apache.tez outputs are allowed for max percent failure feature. Disallowed Output: " + edgeProperty.getEdgeSource().getClassName()); } List<Event> events = new ArrayList<>(); Deflater deflater = TezCommonUtils.newBestCompressionDeflater(); try { ShuffleUtils.generateEventsForNonStartedOutput(events, edgeManager.getNumDestinationConsumerTasks(attempt.getTaskID().getId()), null, false, true, deflater); } catch (Exception e) { throw new TezException(e); } EventMetaData sourceInfo = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, sourceVertex.getName(), getDestinationVertexName(), attempt); List<TezEvent> tezEvents = new ArrayList<>(events.size()); for (Event e : events) { TezEvent tezEvent = new TezEvent(e, sourceInfo); tezEvents.add(tezEvent); } return tezEvents; }
for (TezEvent tezEvent : tezEvents) { String srcVertexName = tezEvent.getSourceInfo().getTaskVertexName(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId();
TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo() .getTaskAttemptID(); int destTaskIndex = destAttemptId.getTaskID().getId(); int srcTaskIndex; int numConsumers;
TezTaskAttemptID srcAttemptId = tezEvent.getSourceInfo() .getTaskAttemptID(); int srcTaskIndex = srcAttemptId.getTaskID().getId();
IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString());
TaskSpec baseTaskSpec = createRemoteTaskSpec(taskId.getId()); boolean fromRecovery = recoveryData == null ? false : recoveryData.getTaskRecoveryData(taskId) != null; eventHandler.handle(new TaskEventScheduleTask(taskId, baseTaskSpec,