private void cleanupQuery(final QueryId queryId) { if (getQuery() != null) { Set<InetSocketAddress> workers = Sets.newHashSet(); for (Stage stage : getQuery().getStages()) { workers.addAll(stage.getAssignedWorkerMap().values()); } LOG.info("Cleanup resources of all workers. Query: " + queryId + ", workers: " + workers.size()); for (final InetSocketAddress worker : workers) { queryMasterContext.getEventExecutor().submit(new Runnable() { @Override public void run() { try { AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub(); tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get()); } catch (Throwable e) { LOG.error(e.getMessage(), e); } } }); } } }
private void cleanupQuery(final QueryId queryId) { if (getQuery() != null) { Set<InetSocketAddress> workers = Sets.newHashSet(); for (Stage stage : getQuery().getStages()) { workers.addAll(stage.getAssignedWorkerMap().values()); } LOG.info("Cleanup resources of all workers. Query: " + queryId + ", workers: " + workers.size()); for (final InetSocketAddress worker : workers) { queryMasterContext.getEventExecutor().submit(new Runnable() { @Override public void run() { try { AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub(); tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get()); } catch (Throwable e) { LOG.error(e.getMessage(), e); } } }); } } }
private void sendStopExecutionBlockEvent(final StopExecutionBlockRequest requestProto) { for (final InetSocketAddress worker : getAssignedWorkerMap().values()) { getContext().getQueryMasterContext().getEventExecutor().submit(new Runnable() { @Override public void run() { try { AsyncRpcClient tajoWorkerRpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.stopExecutionBlock(null, requestProto, NullCallback.get(PrimitiveProtos.BoolProto.class)); } catch (Throwable e) { LOG.error(e.getMessage(), e); } } }); } }
private void sendStopExecutionBlockEvent(final StopExecutionBlockRequest requestProto) { for (final InetSocketAddress worker : getAssignedWorkerMap().values()) { getContext().getQueryMasterContext().getEventExecutor().submit(new Runnable() { @Override public void run() { try { AsyncRpcClient tajoWorkerRpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.stopExecutionBlock(null, requestProto, NullCallback.get(PrimitiveProtos.BoolProto.class)); } catch (Throwable e) { LOG.error(e.getMessage(), e); } } }); } }
@Override public void start() { info(LOG, "Start TaskScheduler"); maximumRequestContainer = Math.min(tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_REQUEST_MAX_NUM) , stage.getContext().getWorkerMap().size()); if (isLeaf) { candidateWorkers.addAll(getWorkerIds(getLeafTaskHosts())); } else { //find assigned hosts for Non-Leaf locality in children executionBlock List<ExecutionBlock> executionBlockList = stage.getMasterPlan().getChilds(stage.getBlock()); for (ExecutionBlock executionBlock : executionBlockList) { Stage childStage = stage.getContext().getStage(executionBlock.getId()); candidateWorkers.addAll(childStage.getAssignedWorkerMap().keySet()); } } this.schedulingThread.start(); super.start(); }
@Override public void start() { info(LOG, "Start TaskScheduler"); maximumRequestContainer = Math.min(tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_REQUEST_MAX_NUM) , stage.getContext().getWorkerMap().size()); if (isLeaf) { candidateWorkers.addAll(getWorkerIds(getLeafTaskHosts())); } else { //find assigned hosts for Non-Leaf locality in children executionBlock List<ExecutionBlock> executionBlockList = stage.getMasterPlan().getChilds(stage.getBlock()); for (ExecutionBlock executionBlock : executionBlockList) { Stage childStage = stage.getContext().getStage(executionBlock.getId()); candidateWorkers.addAll(childStage.getAssignedWorkerMap().keySet()); } } this.schedulingThread.start(); super.start(); }
/** * It sends a kill RPC request to a corresponding worker. * * @param workerId worker unique Id. * @param taskAttemptId The TaskAttemptId to be killed. */ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { NettyClientBase tajoWorkerRpc; ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId(); InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<>(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); if(!callFuture.get().getValue()){ getEventHandler().handle( new TaskFatalErrorEvent(taskAttemptId, new TajoInternalError("Can't kill task :" + taskAttemptId))); } } catch (Exception e) { /* Node RPC failure */ LOG.error(e.getMessage(), e); getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e)); } }
/** * It sends a kill RPC request to a corresponding worker. * * @param workerId worker unique Id. * @param taskAttemptId The TaskAttemptId to be killed. */ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { NettyClientBase tajoWorkerRpc; ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId(); InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<PrimitiveProtos.BoolProto>(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); if(!callFuture.get().getValue()){ getEventHandler().handle( new TaskFatalErrorEvent(taskAttemptId, new TajoInternalError("Can't kill task :" + taskAttemptId))); } } catch (Exception e) { /* Node RPC failure */ LOG.error(e.getMessage(), e); getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e)); } }
context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo)); InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId()); if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());
context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo)); InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId()); if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());
@Override public void getExecutionBlockContext(RpcController controller, ExecutionBlockContextRequest request, RpcCallback<ExecutionBlockContextResponse> done) { QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask( new QueryId(request.getExecutionBlockId().getQueryId())); if (queryMasterTask != null) { Stage stage = queryMasterTask.getQuery().getStage(new ExecutionBlockId(request.getExecutionBlockId())); // first request with starting ExecutionBlock PlanProto.ShuffleType shuffleType = stage.getDataChannel().getShuffleType(); ExecutionBlockContextResponse.Builder ebRequestProto = ExecutionBlockContextResponse.newBuilder(); ebRequestProto.setExecutionBlockId(request.getExecutionBlockId()) .setQueryContext(stage.getContext().getQueryContext().getProto()) .setQueryOutputPath(stage.getContext().getStagingDir().toString()) .setPlanJson(CoreGsonHelper.toJson(stage.getBlock().getPlan(), LogicalNode.class)) .setShuffleType(shuffleType); //Set assigned worker to stage if (!stage.getAssignedWorkerMap().containsKey(request.getWorker().getId())) { stage.getAssignedWorkerMap().put(request.getWorker().getId(), NetUtils.createSocketAddr(request.getWorker().getHost(), request.getWorker().getPeerRpcPort())); } done.run(ebRequestProto.build()); } else { controller.setFailed("Can't find query. request: " + request); } }
@Override public void getExecutionBlockContext(RpcController controller, ExecutionBlockContextRequest request, RpcCallback<ExecutionBlockContextResponse> done) { QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask( new QueryId(request.getExecutionBlockId().getQueryId())); if (queryMasterTask != null) { Stage stage = queryMasterTask.getQuery().getStage(new ExecutionBlockId(request.getExecutionBlockId())); // first request with starting ExecutionBlock PlanProto.ShuffleType shuffleType = stage.getDataChannel().getShuffleType(); ExecutionBlockContextResponse.Builder ebRequestProto = ExecutionBlockContextResponse.newBuilder(); ebRequestProto.setExecutionBlockId(request.getExecutionBlockId()) .setQueryContext(stage.getContext().getQueryContext().getProto()) .setQueryOutputPath(stage.getContext().getStagingDir().toString()) .setPlanJson(CoreGsonHelper.toJson(stage.getBlock().getPlan(), LogicalNode.class)) .setShuffleType(shuffleType); //Set assigned worker to stage if (!stage.getAssignedWorkerMap().containsKey(request.getWorker().getId())) { stage.getAssignedWorkerMap().put(request.getWorker().getId(), NetUtils.createSocketAddr(request.getWorker().getHost(), request.getWorker().getPeerRpcPort())); } done.run(ebRequestProto.build()); } else { controller.setFailed("Can't find query. request: " + request); } }
InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId()); if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());
InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId()); if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());