private AMNodeInfo getAMNodeInfo(String amHost, int amPort, QueryIdentifier queryId) { Map<LlapNodeId, AMNodeInfo> amNodeInfoPerQuery = knownAppMasters.get(queryId); if (amNodeInfoPerQuery != null) { LlapNodeId amNodeId = LlapNodeId.getInstance(amHost, amPort); return amNodeInfoPerQuery.get(amNodeId); } return null; }
void registerContainer(ContainerId containerId, String hostname, int port) { if (LOG.isDebugEnabled()) { LOG.debug("Registering " + containerId + " for node: " + hostname + ":" + port); } containerToNodeMap.putIfAbsent(containerId, LlapNodeId.getInstance(hostname, port)); // nodeMap registration is not required, since there's no taskId association. }
@Override public LlapNodeId getNodeId() throws InterruptedException, TimeoutException { ensureInfo(); return LlapNodeId.getInstance(info.amHost, info.amPluginPort); }
void registerTaskAttempt(ContainerId containerId, TezTaskAttemptID taskAttemptId, String host, int port) { if (LOG.isDebugEnabled()) { LOG.debug("Registering " + containerId + ", " + taskAttemptId + " for node: " + host + ":" + port); } LlapNodeId llapNodeId = LlapNodeId.getInstance(host, port); attemptToNodeMap.putIfAbsent(taskAttemptId, llapNodeId); registerContainer(containerId, host, port); // nodeMap registration. BiMap<ContainerId, TezTaskAttemptID> tmpMap = HashBiMap.create(); BiMap<ContainerId, TezTaskAttemptID> old = nodeMap.putIfAbsent(llapNodeId, tmpMap); BiMap<ContainerId, TezTaskAttemptID> usedInstance; usedInstance = old == null ? tmpMap : old; synchronized(usedInstance) { usedInstance.put(containerId, taskAttemptId); } // Make sure to put the instance back again, in case it was removed as part of a // containerEnd/taskEnd invocation. nodeMap.putIfAbsent(llapNodeId, usedInstance); }
@Override public void serviceStart() { QueueLookupCallable queueDrainerCallable = new QueueLookupCallable(); queueLookupFuture = queueLookupExecutor.submit(queueDrainerCallable); Futures.addCallback(queueLookupFuture, new FutureCallback<Void>() { @Override public void onSuccess(Void result) { LOG.info("AMReporter QueueDrainer exited"); } @Override public void onFailure(Throwable t) { if (t instanceof CancellationException && isShutdown.get()) { LOG.info("AMReporter QueueDrainer exited as a result of a cancellation after shutdown"); } else { LOG.error("AMReporter QueueDrainer exited with error", t); Thread.getDefaultUncaughtExceptionHandler().uncaughtException(Thread.currentThread(), t); } } }); // TODO: why is this needed? we could just save the host and port? nodeId = LlapNodeId.getInstance(localAddress.get().getHostName(), localAddress.get().getPort()); LOG.info("AMReporter running with DaemonId: {}, NodeId: {}", daemonId, nodeId); }
public void sendTerminateFragment(final TerminateFragmentRequestProto request, final String host, final int port, final ExecuteRequestCallback<TerminateFragmentResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new SendTerminateFragmentCallable(nodeId, request, callback)); }
public void sendUpdateFragment(final UpdateFragmentRequestProto request, final String host, final int port, final ExecuteRequestCallback<UpdateFragmentResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new SendUpdateFragmentCallable(nodeId, request, callback)); }
public void sendSubmitWork(SubmitWorkRequestProto request, String host, int port, final ExecuteRequestCallback<SubmitWorkResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new SubmitWorkCallable(nodeId, request, callback)); }
@Override public UpdateQueryResponseProto call() throws Exception { ensureInfo(); LlapNodeId nodeId = LlapNodeId.getInstance(info.amHost, info.amPluginPort); return getProxy(nodeId, info.amPluginToken).updateQuery(null, request); }
public void registerDag(RegisterDagRequestProto request, String host, int port, final ExecuteRequestCallback<RegisterDagResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new RegisterDagCallable(nodeId, request, callback)); }
public void sendQueryComplete(final QueryCompleteRequestProto request, final String host, final int port, final ExecuteRequestCallback<QueryCompleteResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new SendQueryCompleteCallable(nodeId, request, callback)); }
LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port); Map<LlapNodeId, AMNodeInfo> amNodeInfoPerQuery = knownAppMasters.get(queryIdentifier); if (amNodeInfoPerQuery == null) {
public void taskKilled(String amLocation, int port, String umbilicalUser, Token<JobTokenIdentifier> jobToken, final QueryIdentifier queryIdentifier, final TezTaskAttemptID taskAttemptId) { LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port); AMNodeInfo amNodeInfo; synchronized (knownAppMasters) { amNodeInfo = getAMNodeInfo(amLocation, port, queryIdentifier); if (amNodeInfo == null) { amNodeInfo = new AMNodeInfo(amNodeId, umbilicalUser, jobToken, queryIdentifier, retryPolicy, retryTimeout, socketFactory, conf); } } // Even if the service hasn't started up. It's OK to make this invocation since this will // only happen after the AtomicReference address has been populated. Not adding an additional check. ListenableFuture<Void> future = executor.submit(new KillTaskCallable(taskAttemptId, amNodeInfo)); Futures.addCallback(future, new FutureCallback<Void>() { @Override public void onSuccess(Void result) { LOG.info("Sent taskKilled for {}", taskAttemptId); } @Override public void onFailure(Throwable t) { LOG.warn("Failed to send taskKilled for {}. The attempt will likely time out.", taskAttemptId); } }); }
public static QueryInfo createQueryInfo() { QueryIdentifier queryIdentifier = new QueryIdentifier("fake_app_id_string", 1); LlapNodeId nodeId = LlapNodeId.getInstance("localhost", 0); QueryInfo queryInfo = new QueryInfo(queryIdentifier, "fake_app_id_string", "fake_dag_id_string", "fake_dag_name", "fakeHiveQueryId", 1, "fakeUser", new ConcurrentHashMap<String, LlapDaemonProtocolProtos.SourceStateProto>(), new String[0], null, "fakeUser", null, nodeId, null, null, false); return queryInfo; }
void nodePinged(String hostname, String uniqueId, int port, TezAttemptArray tasks, BooleanArray guaranteed) { LlapNodeId nodeId = LlapNodeId.getInstance(hostname, port); registerPingingNode(nodeId); BiMap<ContainerId, TezTaskAttemptID> biMap =
LlapNodeId nodeId = LlapNodeId.getInstance(host, port); NodeInfo nodeInfo = getNodeInfo(nodeId);
LlapNodeId nodeId = LlapNodeId.getInstance(host, port); registerKnownNode(nodeId); entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port);
LlapNodeId amNodeId = LlapNodeId.getInstance(request.getAmHost(), request.getAmPort()); QueryFragmentInfo fragmentInfo = queryTracker.registerFragment( queryIdentifier, qIdProto.getApplicationIdString(), dagId,
if (nodeId == null) { if (assignedNode != null) { nodeId = LlapNodeId.getInstance(assignedNode.getHost(), assignedNode.getRpcPort());
public void sendUpdateFragment(final UpdateFragmentRequestProto request, final String host, final int port, final ExecuteRequestCallback<UpdateFragmentResponseProto> callback) { LlapNodeId nodeId = LlapNodeId.getInstance(host, port); queueRequest(new SendUpdateFragmentCallable(nodeId, request, callback)); }