public static String getTaskIdentifierString( SubmitWorkRequestProto request, SignableVertexSpec vertex, String queryId) { StringBuilder sb = new StringBuilder(); sb.append("AppId=").append(vertex.getQueryIdentifier().getApplicationIdString()) .append(", containerId=").append(request.getContainerIdString()) .append(", QueryId=").append(queryId) .append(", Vertex=").append(vertex.getVertexName()) .append(", FragmentNum=").append(request.getFragmentNumber()) .append(", Attempt=").append(request.getAttemptNumber()); return sb.toString(); }
private String stringifySourceStateUpdateRequest(SourceStateUpdatedRequestProto request) { StringBuilder sb = new StringBuilder(); QueryIdentifier queryIdentifier = new QueryIdentifier(request.getQueryIdentifier().getApplicationIdString(), request.getQueryIdentifier().getDagIndex()); sb.append("queryIdentifier=").append(queryIdentifier) .append(", ").append("sourceName=").append(request.getSrcName()) .append(", ").append("state=").append(request.getState()); return sb.toString(); }
sb.append(", user=").append(vertex.getUser()); sb.append(", queryId=").append(vertex.getHiveQueryId()); sb.append(", appIdString=").append(vertex.getQueryIdentifier().getApplicationIdString()); sb.append(", appAttemptNum=").append(vertex.getQueryIdentifier().getAppAttemptNumber()); sb.append(", containerIdString=").append(request.getContainerIdString());
sb.append(", user=").append(vertex.getUser()); sb.append(", queryId=").append(vertex.getHiveQueryId()); sb.append(", appIdString=").append(vertex.getQueryIdentifier().getApplicationIdString()); sb.append(", appAttemptNum=").append(vertex.getQueryIdentifier().getAppAttemptNumber()); sb.append(", containerIdString=").append(request.getContainerIdString());
@Override public QueryCompleteResponseProto queryComplete( QueryCompleteRequestProto request) throws IOException { QueryIdentifier queryIdentifier = new QueryIdentifier(request.getQueryIdentifier().getApplicationIdString(), request.getQueryIdentifier().getDagIndex()); LOG.info("Processing queryComplete notification for {}", queryIdentifier); QueryInfo queryInfo = queryTracker.queryComplete(queryIdentifier, request.getDeleteDelay(), false); if (queryInfo != null) { List<QueryFragmentInfo> knownFragments = queryInfo.getRegisteredFragments(); LOG.info("DBG: Pending fragment count for completed query {} = {}", queryIdentifier, knownFragments.size()); for (QueryFragmentInfo fragmentInfo : knownFragments) { LOG.info("Issuing killFragment for completed query {} {}", queryIdentifier, fragmentInfo.getFragmentIdentifierString()); executorService.killFragment(fragmentInfo.getFragmentIdentifierString()); } amReporter.queryComplete(queryIdentifier); } return QueryCompleteResponseProto.getDefaultInstance(); }
public static String getTaskIdentifierString( SubmitWorkRequestProto request, SignableVertexSpec vertex, String queryId) { StringBuilder sb = new StringBuilder(); sb.append("AppId=").append(vertex.getQueryIdentifier().getApplicationIdString()) .append(", containerId=").append(request.getContainerIdString()) .append(", QueryId=").append(queryId) .append(", Vertex=").append(vertex.getVertexName()) .append(", FragmentNum=").append(request.getFragmentNumber()) .append(", Attempt=").append(request.getAttemptNumber()); return sb.toString(); }
private String stringifySourceStateUpdateRequest(SourceStateUpdatedRequestProto request) { StringBuilder sb = new StringBuilder(); QueryIdentifier queryIdentifier = new QueryIdentifier(request.getQueryIdentifier().getApplicationIdString(), request.getQueryIdentifier().getDagIndex()); sb.append("queryIdentifier=").append(queryIdentifier) .append(", ").append("sourceName=").append(request.getSrcName()) .append(", ").append("state=").append(request.getState()); return sb.toString(); }
@Override public SourceStateUpdatedResponseProto sourceStateUpdated( SourceStateUpdatedRequestProto request) throws IOException { LOG.info("Processing state update: " + stringifySourceStateUpdateRequest(request)); QueryIdentifier queryId = new QueryIdentifier(request.getQueryIdentifier().getApplicationIdString(), request.getQueryIdentifier().getDagIndex()); queryTracker.registerSourceStateChange(queryId, request.getSrcName(), request.getState()); return SourceStateUpdatedResponseProto.getDefaultInstance(); }
protected void logFragmentEnd(boolean success) { LOG.info("WM counters: {}", wmCounters); HistoryLogger.logFragmentEnd(vertex.getQueryIdentifier().getApplicationIdString(), request.getContainerIdString(), executionContext.getHostName(), queryId, fragmentInfo.getQueryInfo().getDagIdentifier(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), taskRunnerCallable.threadName, taskRunnerCallable.startTime, success); } }
HistoryLogger.logFragmentStart(qIdProto.getApplicationIdString(), request.getContainerIdString(), localAddress.get().getHostName(), constructUniqueQueryId(vertex.getHiveQueryId(), qIdProto.getDagIndex()), qIdProto.getApplicationIdString(), dagIdentifier); queryIdentifier, qIdProto.getApplicationIdString(), dagId, vertex.getDagName(), vertex.getHiveQueryId(), dagIdentifier, vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(),
assertEquals("vertexName", vertexProto.getVertexName()); assertEquals("hiveQueryId", vertexProto.getHiveQueryId()); assertEquals(appId.toString(), vertexProto.getQueryIdentifier().getApplicationIdString()); assertEquals(tezDagId.getId(), vertexProto.getQueryIdentifier().getDagIndex()); assertEquals(333, vertexProto.getQueryIdentifier().getAppAttemptNumber());
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto)) { return super.equals(obj); } org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto) obj; boolean result = true; result = result && (hasApplicationIdString() == other.hasApplicationIdString()); if (hasApplicationIdString()) { result = result && getApplicationIdString() .equals(other.getApplicationIdString()); } result = result && (hasDagIndex() == other.hasDagIndex()); if (hasDagIndex()) { result = result && (getDagIndex() == other.getDagIndex()); } result = result && (hasAppAttemptNumber() == other.hasAppAttemptNumber()); if (hasAppAttemptNumber()) { result = result && (getAppAttemptNumber() == other.getAppAttemptNumber()); } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; }
@java.lang.Override public int hashCode() { if (memoizedHashCode != 0) { return memoizedHashCode; } int hash = 41; hash = (19 * hash) + getDescriptorForType().hashCode(); if (hasApplicationIdString()) { hash = (37 * hash) + APPLICATION_ID_STRING_FIELD_NUMBER; hash = (53 * hash) + getApplicationIdString().hashCode(); } if (hasDagIndex()) { hash = (37 * hash) + DAG_INDEX_FIELD_NUMBER; hash = (53 * hash) + getDagIndex(); } if (hasAppAttemptNumber()) { hash = (37 * hash) + APP_ATTEMPT_NUMBER_FIELD_NUMBER; hash = (53 * hash) + getAppAttemptNumber(); } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; }
@Override public QueryCompleteResponseProto queryComplete( QueryCompleteRequestProto request) throws IOException { QueryIdentifier queryIdentifier = new QueryIdentifier(request.getQueryIdentifier().getApplicationIdString(), request.getQueryIdentifier().getDagIndex()); LOG.info("Processing queryComplete notification for {}", queryIdentifier); QueryInfo queryInfo = queryTracker.queryComplete(queryIdentifier, request.getDeleteDelay(), false); if (queryInfo != null) { List<QueryFragmentInfo> knownFragments = queryInfo.getRegisteredFragments(); LOG.info("DBG: Pending fragment count for completed query {} = {}", queryIdentifier, knownFragments.size()); for (QueryFragmentInfo fragmentInfo : knownFragments) { LOG.info("Issuing killFragment for completed query {} {}", queryIdentifier, fragmentInfo.getFragmentIdentifierString()); executorService.killFragment(fragmentInfo.getFragmentIdentifierString()); } amReporter.queryComplete(queryIdentifier); } return QueryCompleteResponseProto.getDefaultInstance(); }
@Override public RegisterDagResponseProto registerDag(RegisterDagRequestProto request) throws IOException { QueryIdentifierProto identifier = request.getQueryIdentifier(); Credentials credentials; if (request.hasCredentialsBinary()) { credentials = LlapUtil.credentialsFromByteArray( request.getCredentialsBinary().toByteArray()); } else { credentials = new Credentials(); } queryTracker.registerDag(identifier.getApplicationIdString(), identifier.getDagIndex(), request.getUser(), credentials); if (LOG.isInfoEnabled()) { LOG.info("Application with id={}, dagId={} registered", identifier.getApplicationIdString(), identifier.getDagIndex()); } return RegisterDagResponseProto.newBuilder().build(); }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }
@Override public void setDone(Void v, QueryIdentifierProto result) { LOG.info("Dag with" + " appId=" + result.getApplicationIdString() + " dagId=" + result.getDagIndex() + " registered successfully for node " + nodeInfo.getHost()); addNode(nodeInfo, llapServiceInstance); }
protected void logFragmentEnd(boolean success) { LOG.info("WM counters: {}", wmCounters); HistoryLogger.logFragmentEnd(vertex.getQueryIdentifier().getApplicationIdString(), request.getContainerIdString(), executionContext.getHostName(), queryId, fragmentInfo.getQueryInfo().getDagIdentifier(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), taskRunnerCallable.threadName, taskRunnerCallable.startTime, success); } }
/** * <code>optional string application_id_string = 1;</code> */ public Builder clearApplicationIdString() { bitField0_ = (bitField0_ & ~0x00000001); applicationIdString_ = getDefaultInstance().getApplicationIdString(); onChanged(); return this; } /**
@Override public SourceStateUpdatedResponseProto sourceStateUpdated( SourceStateUpdatedRequestProto request) throws IOException { LOG.info("Processing state update: " + stringifySourceStateUpdateRequest(request)); QueryIdentifier queryId = new QueryIdentifier(request.getQueryIdentifier().getApplicationIdString(), request.getQueryIdentifier().getDagIndex()); queryTracker.registerSourceStateChange(queryId, request.getSrcName(), request.getState()); return SourceStateUpdatedResponseProto.getDefaultInstance(); }