private AMNodeInfo getAMNodeInfo(String amHost, int amPort, QueryIdentifier queryId) { Map<LlapNodeId, AMNodeInfo> amNodeInfoPerQuery = knownAppMasters.get(queryId); if (amNodeInfoPerQuery != null) { LlapNodeId amNodeId = LlapNodeId.getInstance(amHost, amPort); return amNodeInfoPerQuery.get(amNodeId); } return null; }
@Override public ProtocolType run() { return createProtocolImpl(getConfig(), nodeId.getHostname(), nodeId.getPort(), ugi, retryPolicy, socketFactory); } });
public static LlapNodeId getInstance(String hostname, int port) { return CACHE.getUnchecked(new LlapNodeId(hostname, port)); }
if (nodeId == null) { if (assignedNode != null) { nodeId = LlapNodeId.getInstance(assignedNode.getHost(), assignedNode.getRpcPort()); attemptId.getTaskID().getVertexID().getDAGId().getId())).build(); communicator.sendUpdateFragment(request, nodeId.getHostname(), nodeId.getPort(), new LlapProtocolClientProxy.ExecuteRequestCallback<UpdateFragmentResponseProto>() { @Override
if (nodeId == null) { if (assignedNode != null) { nodeId = LlapNodeId.getInstance(assignedNode.getHost(), assignedNode.getRpcPort()); attemptId.getTaskID().getVertexID().getDAGId().getId())).build(); communicator.sendUpdateFragment(request, nodeId.getHostname(), nodeId.getPort(), new LlapProtocolClientProxy.ExecuteRequestCallback<UpdateFragmentResponseProto>() { @Override
void registerContainer(ContainerId containerId, String hostname, int port) { if (LOG.isDebugEnabled()) { LOG.debug("Registering " + containerId + " for node: " + hostname + ":" + port); } containerToNodeMap.putIfAbsent(containerId, LlapNodeId.getInstance(hostname, port)); // nodeMap registration is not required, since there's no taskId association. }
/** * @param nodeId Hostname + post. * @param nodeToken A custom node token. If not specified, the default token is used. * @return the protocol client implementation for the node. */ protected final ProtocolType getProxy( final LlapNodeId nodeId, final Token<TokenType> nodeToken) { String hostId = getHostIdentifier(nodeId.getHostname(), nodeId.getPort()); try { return hostProxies.get(hostId, new Callable<ProtocolType>() { @Override public ProtocolType call() throws Exception { return createProxy(nodeId, nodeToken); } }); } catch (ExecutionException e) { throw new RuntimeException(e); } }
@Override public LlapNodeId getNodeId() throws InterruptedException, TimeoutException { ensureInfo(); return LlapNodeId.getInstance(info.amHost, info.amPluginPort); }
this.isExternalQuery = isExternalQuery; final InetSocketAddress address = NetUtils.createSocketAddrForHost(amNodeId.getHostname(), amNodeId.getPort()); SecurityUtil.setTokenService(appToken, address);
void registerTaskAttempt(ContainerId containerId, TezTaskAttemptID taskAttemptId, String host, int port) { if (LOG.isDebugEnabled()) { LOG.debug("Registering " + containerId + ", " + taskAttemptId + " for node: " + host + ":" + port); } LlapNodeId llapNodeId = LlapNodeId.getInstance(host, port); attemptToNodeMap.putIfAbsent(taskAttemptId, llapNodeId); registerContainer(containerId, host, port); // nodeMap registration. BiMap<ContainerId, TezTaskAttemptID> tmpMap = HashBiMap.create(); BiMap<ContainerId, TezTaskAttemptID> old = nodeMap.putIfAbsent(llapNodeId, tmpMap); BiMap<ContainerId, TezTaskAttemptID> usedInstance; usedInstance = old == null ? tmpMap : old; synchronized(usedInstance) { usedInstance.put(containerId, taskAttemptId); } // Make sure to put the instance back again, in case it was removed as part of a // containerEnd/taskEnd invocation. nodeMap.putIfAbsent(llapNodeId, usedInstance); }
protected LlapTaskUmbilicalProtocol createUmbilical(final AMNodeInfo amNodeInfo) throws IOException, InterruptedException { final InetSocketAddress address = NetUtils.createSocketAddrForHost( amNodeInfo.amNodeId.getHostname(), amNodeInfo.amNodeId.getPort()); SecurityUtil.setTokenService(amNodeInfo.jobToken, address); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(amNodeInfo.umbilicalUser); ugi.addToken(amNodeInfo.jobToken); return ugi.doAs(new PrivilegedExceptionAction<LlapTaskUmbilicalProtocol>() { @Override public LlapTaskUmbilicalProtocol run() throws Exception { return RPC .getProxy(LlapTaskUmbilicalProtocol.class, LlapTaskUmbilicalProtocol.versionID, address, UserGroupInformation.getCurrentUser(), amNodeInfo.conf, amNodeInfo.socketFactory, (int) (amNodeInfo.timeout)); } }); }
@Override public void serviceStart() { QueueLookupCallable queueDrainerCallable = new QueueLookupCallable(); queueLookupFuture = queueLookupExecutor.submit(queueDrainerCallable); Futures.addCallback(queueLookupFuture, new FutureCallback<Void>() { @Override public void onSuccess(Void result) { LOG.info("AMReporter QueueDrainer exited"); } @Override public void onFailure(Throwable t) { if (t instanceof CancellationException && isShutdown.get()) { LOG.info("AMReporter QueueDrainer exited as a result of a cancellation after shutdown"); } else { LOG.error("AMReporter QueueDrainer exited with error", t); Thread.getDefaultUncaughtExceptionHandler().uncaughtException(Thread.currentThread(), t); } } }); // TODO: why is this needed? we could just save the host and port? nodeId = LlapNodeId.getInstance(localAddress.get().getHostName(), localAddress.get().getPort()); LOG.info("AMReporter running with DaemonId: {}, NodeId: {}", daemonId, nodeId); }
LOG.debug("Creating a client without a token for " + nodeId); return createProtocolImpl(getConfig(), nodeId.getHostname(), nodeId.getPort(), null, retryPolicy, socketFactory); nodeId.getHostname(), nodeId.getPort())); ugi.addToken(nodeToken); if (LOG.isDebugEnabled()) {
public void sendTerminateFragment(final TerminateFragmentRequestProto request, final String host, final int port, final ExecuteRequestCallback<TerminateFragmentResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new SendTerminateFragmentCallable(nodeId, request, callback)); }
@Override public void dagComplete(final int dagIdentifier) { QueryIdentifierProto queryIdentifierProto = constructQueryIdentifierProto(dagIdentifier); QueryCompleteRequestProto request = QueryCompleteRequestProto.newBuilder() .setQueryIdentifier(queryIdentifierProto).setDeleteDelay(deleteDelayOnDagComplete).build(); for (final LlapNodeId llapNodeId : nodesForQuery) { LOG.info("Sending dagComplete message for {}, to {}", dagIdentifier, llapNodeId); communicator.sendQueryComplete(request, llapNodeId.getHostname(), llapNodeId.getPort(), new LlapProtocolClientProxy.ExecuteRequestCallback<LlapDaemonProtocolProtos.QueryCompleteResponseProto>() { @Override public void setResponse(LlapDaemonProtocolProtos.QueryCompleteResponseProto response) { } @Override public void indicateError(Throwable t) { LOG.warn("Failed to indicate dag complete dagId={} to node {}", dagIdentifier, llapNodeId); processSendError(t); } }); } nodesForQuery.clear(); // TODO Ideally move some of the other cleanup code from resetCurrentDag over here }
public void sendUpdateFragment(final UpdateFragmentRequestProto request, final String host, final int port, final ExecuteRequestCallback<UpdateFragmentResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new SendUpdateFragmentCallable(nodeId, request, callback)); }
guaranteed.set(tasks.guaranteed.toArray(new BooleanWritable[tasks.guaranteed.size()])); amNodeInfo.getUmbilical().nodeHeartbeat(new Text(nodeId.getHostname()), new Text(daemonId.getUniqueNodeIdInCluster()), nodeId.getPort(), aw, guaranteed); } catch (IOException e) { QueryIdentifier currentQueryIdentifier = amNodeInfo.getQueryIdentifier();
@Override public UpdateQueryResponseProto call() throws Exception { ensureInfo(); LlapNodeId nodeId = LlapNodeId.getInstance(info.amHost, info.amPluginPort); return getProxy(nodeId, info.amPluginToken).updateQuery(null, request); }
taskAttemptId.getTaskID().getVertexID().getDAGId().getId())) .setFragmentIdentifierString(taskAttemptId.toString()).build(); communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(), new LlapProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() { @Override
public void registerDag(RegisterDagRequestProto request, String host, int port, final ExecuteRequestCallback<RegisterDagResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new RegisterDagCallable(nodeId, request, callback)); }