public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec other) { if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec.getDefaultInstance()) return this; if (other.hasUser()) { bitField0_ |= 0x00000001; user_ = other.user_; onChanged(); if (other.hasSignatureKeyId()) { setSignatureKeyId(other.getSignatureKeyId()); if (other.hasQueryIdentifier()) { mergeQueryIdentifier(other.getQueryIdentifier()); if (other.hasHiveQueryId()) { bitField0_ |= 0x00000008; hiveQueryId_ = other.hiveQueryId_; onChanged(); if (other.hasDagName()) { bitField0_ |= 0x00000010; dagName_ = other.dagName_; onChanged(); if (other.hasVertexName()) { bitField0_ |= 0x00000020; vertexName_ = other.vertexName_; onChanged(); if (other.hasVertexIndex()) { setVertexIndex(other.getVertexIndex());
Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); String fragmentIdString = attemptId.toString(); fragmentIdString, stringifySubmitRequest(request, vertex)); QueryIdentifierProto qIdProto = vertex.getQueryIdentifier(); constructUniqueQueryId(vertex.getHiveQueryId(), qIdProto.getDagIndex()), qIdProto.getDagIndex(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber()); final String queryId = vertex.getHiveQueryId(); final String fragmentId = LlapTezUtils.stripAttemptPrefix(fragmentIdString); MDC.put("dagId", dagId); env.put(ApplicationConstants.Environment.USER.name(), vertex.getUser()); QueryFragmentInfo fragmentInfo = queryTracker.registerFragment( queryIdentifier, qIdProto.getApplicationIdString(), dagId, vertex.getDagName(), vertex.getHiveQueryId(), dagIdentifier, vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo, amNodeId);
hash = (19 * hash) + getDescriptorForType().hashCode(); if (hasUser()) { hash = (37 * hash) + USER_FIELD_NUMBER; hash = (53 * hash) + getUser().hashCode(); if (hasSignatureKeyId()) { hash = (37 * hash) + SIGNATUREKEYID_FIELD_NUMBER; hash = (53 * hash) + hashLong(getSignatureKeyId()); if (hasQueryIdentifier()) { hash = (37 * hash) + QUERY_IDENTIFIER_FIELD_NUMBER; hash = (53 * hash) + getQueryIdentifier().hashCode(); if (hasHiveQueryId()) { hash = (37 * hash) + HIVE_QUERY_ID_FIELD_NUMBER; hash = (53 * hash) + getHiveQueryId().hashCode(); if (hasDagName()) { hash = (37 * hash) + DAG_NAME_FIELD_NUMBER; hash = (53 * hash) + getDagName().hashCode(); if (hasVertexName()) { hash = (37 * hash) + VERTEX_NAME_FIELD_NUMBER; hash = (53 * hash) + getVertexName().hashCode(); if (hasVertexIndex()) { hash = (37 * hash) + VERTEX_INDEX_FIELD_NUMBER; hash = (53 * hash) + getVertexIndex();
fsTaskUgi = UserGroupInformation.createRemoteUser(vertex.getUser()); if (!vertex.getIsExternalSubmission()) { taskOwner = fragmentInfo.getQueryInfo().getUmbilicalUgi(); } else { taskOwner = UserGroupInformation.createRemoteUser(vertex.getTokenIdentifier()); taskOwner.addToken(jobToken); final InetSocketAddress address = if (shouldRunTask) { taskRunner = new TezTaskRunner2(conf, fsTaskUgi, fragmentInfo.getLocalDirs(), taskSpec, vertex.getQueryIdentifier().getAppAttemptNumber(), serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memoryAvailable, false, tezHadoopShim);
Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); String fragmentIdString = attemptId.toString(); fragmentIdString, stringifySubmitRequest(request, vertex)); QueryIdentifierProto qIdProto = vertex.getQueryIdentifier(); constructUniqueQueryId(vertex.getHiveQueryId(), qIdProto.getDagIndex()), qIdProto.getDagIndex(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber()); final String queryId = vertex.getHiveQueryId(); final String fragmentId = LlapTezUtils.stripAttemptPrefix(fragmentIdString); MDC.put("dagId", dagId); env.put(ApplicationConstants.Environment.USER.name(), vertex.getUser()); QueryFragmentInfo fragmentInfo = queryTracker.registerFragment( queryIdentifier, qIdProto.getApplicationIdString(), dagId, vertex.getDagName(), vertex.getHiveQueryId(), dagIdentifier, vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo, amNodeId);
SignableVertexSpec vertex = SignableVertexSpec.parseFrom(submitWorkInfo.getVertexBinary()); Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()).toString(); OutputStream socketStream = socket.getOutputStream();
SignableVertexSpec fs = c.getVertexSpec(); value.append(isFirst ? " (" : ", ").append(c.getQueryId()) .append("/").append(fs.getVertexName()).append(c.isGuaranteed() ? ", guaranteed" : ""); isFirst = false;
sb.append(", taskInfo=").append(" fragment ") .append(request.getFragmentNumber()).append(" attempt ").append(request.getAttemptNumber()); sb.append(", user=").append(vertex.getUser()); sb.append(", queryId=").append(vertex.getHiveQueryId()); sb.append(", appIdString=").append(vertex.getQueryIdentifier().getApplicationIdString()); sb.append(", appAttemptNum=").append(vertex.getQueryIdentifier().getAppAttemptNumber()); sb.append(", containerIdString=").append(request.getContainerIdString()); sb.append(", dagName=").append(vertex.getDagName()); sb.append(", vertexName=").append(vertex.getVertexName()); sb.append(", processor=").append(vertex.getProcessorDescriptor().getClassName()); sb.append(", numInputs=").append(vertex.getInputSpecsCount()); sb.append(", numOutputs=").append(vertex.getOutputSpecsCount()); sb.append(", numGroupedInputs=").append(vertex.getGroupedInputSpecsCount()); sb.append(", Inputs={"); if (vertex.getInputSpecsCount() > 0) { for (IOSpecProto ioSpec : vertex.getInputSpecsList()) { sb.append("{").append(ioSpec.getConnectedVertexName()).append(",") .append(ioSpec.getIoDescriptor().getClassName()).append(",") if (vertex.getOutputSpecsCount() > 0) { for (IOSpecProto ioSpec : vertex.getOutputSpecsList()) { sb.append("{").append(ioSpec.getConnectedVertexName()).append(",") .append(ioSpec.getIoDescriptor().getClassName()).append(",") if (vertex.getGroupedInputSpecsCount() > 0) { for (GroupInputSpecProto group : vertex.getGroupedInputSpecsList()) { sb.append("{").append("groupName=").append(group.getGroupName()).append(", elements=") .append(group.getGroupVerticesList()).append("}");
/** * Submit the work for actual execution. */ public void submitWork(SubmitWorkRequestProto request, String llapHost, int llapPort) { // Register the pending events to be sent for this spec. VertexOrBinary vob = request.getWorkSpec(); assert vob.hasVertexBinary() != vob.hasVertex(); SignableVertexSpec vertex = null; try { vertex = vob.hasVertex() ? vob.getVertex() : SignableVertexSpec.parseFrom(vob.getVertexBinary()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } QueryIdentifierProto queryIdentifierProto = vertex.getQueryIdentifier(); TezTaskAttemptID attemptId = Converters.createTaskAttemptId(queryIdentifierProto, vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); final String fragmentId = attemptId.toString(); this.requestInfo = new RequestInfo(request, queryIdentifierProto, fragmentId, llapHost, llapPort); this.tezEvents = Lists.<TezEvent>newArrayList(); registerClient(); // Send out the actual SubmitWorkRequest final LlapTaskUmbilicalExternalClient client = this; communicator.start(); submitWork(); }
private SignableVertexSpec extractVertexSpec(SubmitWorkRequestProto request, LlapTokenInfo tokenInfo) throws InvalidProtocolBufferException, IOException { VertexOrBinary vob = request.getWorkSpec(); SignableVertexSpec vertex = vob.hasVertex() ? vob.getVertex() : null; ByteString vertexBinary = vob.hasVertexBinary() ? vob.getVertexBinary() : null; if (vertexBinary != null) { if (vertex != null) { throw new IOException( "Vertex and vertexBinary in VertexOrBinary cannot be set at the same time"); } vertex = SignableVertexSpec.parseFrom(vob.getVertexBinary()); } if (tokenInfo.isSigningRequired) { checkSignature(vertex, vertexBinary, request, tokenInfo.userName); } return vertex; }
private static Multimap<String, String> createStartedInputMap(SignableVertexSpec vertex) { Multimap<String, String> startedInputMap = HashMultimap.create(); // Let the Processor control start for Broadcast inputs. // TODO For now, this affects non broadcast unsorted cases as well. Make use of the edge // property when it's available. for (IOSpecProto inputSpec : vertex.getInputSpecsList()) { if (inputSpec.getIoDescriptor().getClassName().equals(UnorderedKVInput.class.getName())) { startedInputMap.put(vertex.getVertexName(), inputSpec.getConnectedVertexName()); } } return startedInputMap; }
protected void logFragmentEnd(boolean success) { LOG.info("WM counters: {}", wmCounters); HistoryLogger.logFragmentEnd(vertex.getQueryIdentifier().getApplicationIdString(), request.getContainerIdString(), executionContext.getHostName(), queryId, fragmentInfo.getQueryInfo().getDagIdentifier(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), taskRunnerCallable.threadName, taskRunnerCallable.startTime, success); } }
public static String getTaskIdentifierString( SubmitWorkRequestProto request, SignableVertexSpec vertex, String queryId) { StringBuilder sb = new StringBuilder(); sb.append("AppId=").append(vertex.getQueryIdentifier().getApplicationIdString()) .append(", containerId=").append(request.getContainerIdString()) .append(", QueryId=").append(queryId) .append(", Vertex=").append(vertex.getVertexName()) .append(", FragmentNum=").append(request.getFragmentNumber()) .append(", Attempt=").append(request.getAttemptNumber()); return sb.toString(); }
fsTaskUgi = UserGroupInformation.createRemoteUser(vertex.getUser()); if (!vertex.getIsExternalSubmission()) { taskOwner = fragmentInfo.getQueryInfo().getUmbilicalUgi(); } else { taskOwner = UserGroupInformation.createRemoteUser(vertex.getTokenIdentifier()); taskOwner.addToken(jobToken); final InetSocketAddress address = if (shouldRunTask) { taskRunner = new TezTaskRunner2(conf, fsTaskUgi, fragmentInfo.getLocalDirs(), taskSpec, vertex.getQueryIdentifier().getAppAttemptNumber(), serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memoryAvailable, false, tezHadoopShim);
SignableVertexSpec vertex = SignableVertexSpec.parseFrom(submitWorkInfo.getVertexBinary()); Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()).toString(); OutputStream socketStream = socket.getOutputStream();
result = result && (hasUser() == other.hasUser()); if (hasUser()) { result = result && getUser() .equals(other.getUser()); result = result && (hasSignatureKeyId() == other.hasSignatureKeyId()); if (hasSignatureKeyId()) { result = result && (getSignatureKeyId() == other.getSignatureKeyId()); result = result && (hasQueryIdentifier() == other.hasQueryIdentifier()); if (hasQueryIdentifier()) { result = result && getQueryIdentifier() .equals(other.getQueryIdentifier()); result = result && (hasHiveQueryId() == other.hasHiveQueryId()); if (hasHiveQueryId()) { result = result && getHiveQueryId() .equals(other.getHiveQueryId()); result = result && (hasDagName() == other.hasDagName()); if (hasDagName()) { result = result && getDagName() .equals(other.getDagName()); result = result && (hasVertexName() == other.hasVertexName()); if (hasVertexName()) { result = result && getVertexName() .equals(other.getVertexName());
.setAppAttemptNumber(333).setDagIndex(tezDagId.getId()).build(); SignableVertexSpec.Builder builder = SignableVertexSpec.newBuilder(); builder.setQueryIdentifier(queryIdentifierProto); builder.setHiveQueryId("hiveQueryId"); assertEquals(2, taskSpec.getOutputs().size()); verifyInputSpecAndProto(taskSpec.getInputs().get(0), vertexProto.getInputSpecs(0)); verifyInputSpecAndProto(taskSpec.getInputs().get(1), vertexProto.getInputSpecs(1)); verifyOutputSpecAndProto(taskSpec.getOutputs().get(0), vertexProto.getOutputSpecs(0)); verifyOutputSpecAndProto(taskSpec.getOutputs().get(1), vertexProto.getOutputSpecs(1));
.setWorkSpec( VertexOrBinary.newBuilder().setVertex( SignableVertexSpec.newBuilder() .setDagName(dagName) .setHiveQueryId(dagName)
SignableVertexSpec fs = c.getVertexSpec(); value.append(isFirst ? " (" : ", ").append(c.getQueryId()) .append("/").append(fs.getVertexName()).append(c.isGuaranteed() ? ", guaranteed" : ""); isFirst = false;
LlapDaemonProtocolProtos.VertexOrBinary.newBuilder().setVertex( LlapDaemonProtocolProtos.SignableVertexSpec .newBuilder() .setQueryIdentifier( LlapDaemonProtocolProtos.QueryIdentifierProto.newBuilder()