@Override public void handle(QueryStartEvent event) { LOG.info("Start QueryStartEventHandler:" + event.getQueryId()); QueryMasterTask queryMasterTask = new QueryMasterTask(queryMasterContext, event.getQueryId(), event.getSession(), event.getQueryContext(), event.getJsonExpr(), event.getAllocation()); synchronized(queryMasterTasks) { queryMasterTasks.put(event.getQueryId(), queryMasterTask); } queryMasterTask.init(systemConf); if (!queryMasterTask.isInitError()) { queryMasterTask.start(); } queryContext = event.getQueryContext(); if (queryMasterTask.isInitError()) { queryMasterContext.stopQuery(queryMasterTask.getQueryId()); } } }
public void handleTaskFailed(TaskFatalErrorReport report) { synchronized(diagnostics) { if (diagnostics.size() < 10) { diagnostics.add(report); } } getEventHandler().handle(new TaskFatalErrorEvent(report)); }
private TajoHeartbeatRequest buildTajoHeartBeat(QueryMasterTask queryMasterTask) { TajoHeartbeatRequest.Builder builder = TajoHeartbeatRequest.newBuilder(); builder.setConnectionInfo(workerContext.getConnectionInfo().getProto()); builder.setQueryId(queryMasterTask.getQueryId().getProto()); builder.setState(queryMasterTask.getState()); if (queryMasterTask.getQuery() != null) { if (queryMasterTask.getQuery().getResultDesc() != null) { builder.setResultDesc(queryMasterTask.getQuery().getResultDesc().getProto()); } builder.setQueryProgress(queryMasterTask.getQuery().getProgress()); if(queryMasterTask.getQuery().getFailureReason() != null) { builder.setError(queryMasterTask.getQuery().getFailureReason()); } } if (queryMasterTask.isInitError()) { builder.setError(ErrorUtil.convertException(queryMasterTask.getInitError())); } return builder.build(); }
public void run() { LOG.info("ClientSessionTimeoutCheckThread started"); while(!isStopped) { try { synchronized (this) { this.wait(1000); } } catch (InterruptedException e) { break; } List<QueryMasterTask> tempTasks = new ArrayList<QueryMasterTask>(); tempTasks.addAll(queryMasterTasks.values()); for(QueryMasterTask eachTask: tempTasks) { if(!eachTask.isStopped()) { try { long lastHeartbeat = eachTask.getLastClientHeartbeat(); long time = System.currentTimeMillis() - lastHeartbeat; if(lastHeartbeat > 0 && time > querySessionTimeout * 1000) { LOG.warn("Query " + eachTask.getQueryId() + " stopped cause query session timeout: " + time + " ms"); eachTask.expireQuerySession(); } } catch (Exception e) { LOG.error(eachTask.getQueryId() + ":" + e.getMessage(), e); } } } } } }
QueryMasterTask queryMasterTask = new QueryMasterTask(qm.getContext(), queryId, session, defaultContext, expr.toJson(), NodeResources.createResource(512), dispatch); queryMasterTask.init(conf); queryMasterTask.getQueryTaskContext().getDispatcher().start(); queryMasterTask.startQuery(); fail("Query state : " + queryMasterTask.getQuery().getSynchronizedState()); Stage stage = queryMasterTask.getQuery().getStages().iterator().next(); assertNotNull(stage); queryMasterTask.getEventHandler().handle(new QueryEvent(queryId, QueryEventType.KILL)); cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 50); assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState()); } catch (Exception e) { e.printStackTrace(); queryMasterTask.stop();
/** * It sends a kill RPC request to a corresponding worker. * * @param workerId worker unique Id. * @param taskAttemptId The TaskAttemptId to be killed. */ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { NettyClientBase tajoWorkerRpc; ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId(); InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<PrimitiveProtos.BoolProto>(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); if(!callFuture.get().getValue()){ getEventHandler().handle( new TaskFatalErrorEvent(taskAttemptId, new TajoInternalError("Can't kill task :" + taskAttemptId))); } } catch (Exception e) { /* Node RPC failure */ LOG.error(e.getMessage(), e); getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e)); } }
@Override public void serviceStop() throws Exception { isStopped = true; LOG.info("Stopping QueryMasterTask:" + queryId); //release QM resource EventHandler handler = getQueryTaskContext().getQueryMasterContext().getWorkerContext(). getNodeResourceManager().getDispatcher().getEventHandler(); handler.handle(new NodeResourceDeallocateEvent(allocation, NodeResourceEvent.ResourceType.QUERY_MASTER)); //flush current node resource handler.handle(new NodeStatusEvent(NodeStatusEvent.EventType.FLUSH_REPORTS)); if (!queryContext.getBool(SessionVars.DEBUG_ENABLED)) { cleanupQuery(getQueryId()); } super.serviceStop(); LOG.info("Stopped QueryMasterTask:" + queryId); }
public Query getQuery(QueryId queryId) { return queryMasterTasks.get(queryId).getQuery(); }
CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog, TablespaceManager.getInstance()); space.rewritePlan(queryContext, plan); initStagingDir();
@Override public int compare(QueryMasterTask task1, QueryMasterTask task2) { if(desc) { return task2.getQueryId().toString().compareTo(task1.getQueryId().toString()); } else { return task1.getQueryId().toString().compareTo(task2.getQueryId().toString()); } } });
@Override public void serviceStart() throws Exception { startQuery(); List<TajoProtos.WorkerConnectionInfoProto> workersProto = queryMasterContext.getQueryMaster().getAllWorker(); for (TajoProtos.WorkerConnectionInfoProto worker : workersProto) { workerMap.put(worker.getId(), new WorkerConnectionInfo(worker)); } super.serviceStart(); }
public void waitForQuerySubmitted(QueryId queryId, int delay) throws Exception { QueryMasterTask qmt = null; int i = 0; while (qmt == null || TajoClientUtil.isQueryWaitingForSchedule(qmt.getState())) { try { Thread.sleep(delay); if (qmt == null) { qmt = getQueryMasterTask(queryId); } } catch (InterruptedException e) { } if (++i > 200) { throw new IOException("Timed out waiting for query to start"); } } }
@Override public void serviceInit(Configuration conf) throws Exception { systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); rpcParams = RpcParameterFactory.get(systemConf); queryTaskContext = new QueryMasterTaskContext(); addService(dispatcher); dispatcher.register(StageEventType.class, new StageEventDispatcher()); dispatcher.register(TaskEventType.class, new TaskEventDispatcher()); dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher()); dispatcher.register(QueryMasterQueryCompletedEvent.EventType.class, new QueryFinishEventHandler()); dispatcher.register(TaskSchedulerEvent.EventType.class, new TaskSchedulerDispatcher()); dispatcher.register(LocalTaskEventType.class, new LocalTaskEventHandler()); super.serviceInit(systemConf); }
QueryMasterTask queryMasterTask = new QueryMasterTask(qm.getContext(), queryId, session, defaultContext, expr.toJson(), NodeResources.createResource(512), dispatch); queryMasterTask.init(conf); queryMasterTask.getQueryTaskContext().getDispatcher().start(); queryMasterTask.startQuery(); fail("Query state : " + queryMasterTask.getQuery().getSynchronizedState()); Stage stage = queryMasterTask.getQuery().getStages().iterator().next(); assertNotNull(stage); queryMasterTask.getEventHandler().handle(new QueryEvent(queryId, QueryEventType.KILL)); cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 50); assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState()); } finally { queryMasterTask.stop(); List<Stage> stages = Lists.newArrayList(queryMasterTask.getQuery().getStages()); Stage lastStage = stages.get(stages.size() - 1);
/** * It sends a kill RPC request to a corresponding worker. * * @param workerId worker unique Id. * @param taskAttemptId The TaskAttemptId to be killed. */ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { NettyClientBase tajoWorkerRpc; ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId(); InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<>(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); if(!callFuture.get().getValue()){ getEventHandler().handle( new TaskFatalErrorEvent(taskAttemptId, new TajoInternalError("Can't kill task :" + taskAttemptId))); } } catch (Exception e) { /* Node RPC failure */ LOG.error(e.getMessage(), e); getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e)); } }
@Override public void serviceStop() throws Exception { isStopped = true; LOG.info("Stopping QueryMasterTask:" + queryId); //release QM resource EventHandler handler = getQueryTaskContext().getQueryMasterContext().getWorkerContext(). getNodeResourceManager().getDispatcher().getEventHandler(); handler.handle(new NodeResourceDeallocateEvent(new Allocation(allocation), NodeResourceEvent.ResourceType.QUERY_MASTER)); //flush current node resource handler.handle(new NodeStatusEvent(NodeStatusEvent.EventType.FLUSH_REPORTS)); if (!queryContext.getBool(SessionVars.DEBUG_ENABLED)) { cleanupQuery(getQueryId()); } super.serviceStop(); LOG.info("Stopped QueryMasterTask:" + queryId); }
public void run() { LOG.info("ClientSessionTimeoutCheckThread started"); while(!isStopped) { try { synchronized (this) { this.wait(1000); } } catch (InterruptedException e) { break; } List<QueryMasterTask> tempTasks = new ArrayList<>(); tempTasks.addAll(queryMasterTasks.values()); for(QueryMasterTask eachTask: tempTasks) { if(!eachTask.isStopped()) { try { long lastHeartbeat = eachTask.getLastClientHeartbeat(); long time = System.currentTimeMillis() - lastHeartbeat; if(lastHeartbeat > 0 && time > querySessionTimeout * 1000) { LOG.warn("Query " + eachTask.getQueryId() + " stopped cause query session timeout: " + time + " ms"); eachTask.expireQuerySession(); } } catch (Exception e) { LOG.error(eachTask.getQueryId() + ":" + e.getMessage(), e); } } } } } }
public Query getQuery(QueryId queryId) { return queryMasterTasks.get(queryId).getQuery(); }