public Query(final QueryMasterTask.QueryMasterTaskContext context, final QueryId id, final long appSubmitTime, final String queryStr, final EventHandler eventHandler, final MasterPlan plan) { this.context = context; this.systemConf = context.getConf(); this.id = id; this.clock = context.getClock(); this.appSubmitTime = appSubmitTime; this.queryStr = queryStr; this.stages = Maps.newConcurrentMap(); this.eventHandler = eventHandler; this.plan = plan; this.cursor = new ExecutionBlockCursor(plan, true); StringBuilder sb = new StringBuilder("\n======================================================="); sb.append("\nThe order of execution: \n"); int order = 1; for (ExecutionBlock currentEB : cursor) { sb.append("\n").append(order).append(": ").append(currentEB.getId()); order++; } sb.append("\n======================================================="); LOG.info(sb); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); stateMachine = stateMachineFactory.make(this); queryState = stateMachine.getCurrentState(); }
public Stage(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, ExecutionBlock block) { this.context = context; this.masterPlan = masterPlan; this.block = block; this.eventHandler = context.getEventHandler(); this.rpcParams = RpcParameterFactory.get(context.getConf()); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); stateMachine = stateMachineFactory.make(this); stageState = stateMachine.getCurrentState(); }
taskVolume = stage.getContext().getQueryContext().getLong(SessionVars.GROUPBY_TASK_INPUT_SIZE); } else if (sortNode != null && groupbyNode == null) { taskVolume = stage.getContext().getQueryContext().getLong(SessionVars.SORT_TASK_INPUT_SIZE); } else if (sortNode != null /* && groupbyNode != null */) { taskVolume = stage.getContext().getQueryContext().getLong(SessionVars.GROUPBY_TASK_INPUT_SIZE); } else { taskVolume = stage.getContext().getQueryContext().getLong(SessionVars.SORT_TASK_INPUT_SIZE); LOG.info(stage.getId() + ", Table's volume is approximately " + mb + " MB"); int minTaskNum = Math.max(1, stage.getContext().getQueryMasterContext().getConf(). getInt(ConfVars.$TEST_MIN_TASK_NUM.varname, 1)); int maxTaskNum = Math.max(minTaskNum, (int) Math.ceil((double)mb / taskVolume));
TableDesc tableDesc = masterContext.getTableDesc(scans[i]); for (Map.Entry<ExecutionBlockId, ExecutionBlockId> unionScanEntry: execBlock.getUnionScanMap().entrySet()) { ExecutionBlockId originScanEbId = unionScanEntry.getKey(); stats[i] += masterContext.getStage(originScanEbId).getResultStats().getNumBytes(); stats[i] = masterContext.getStage(scanEBId).getResultStats().getNumBytes(); StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); String intermediateDataFormat = schedulerContext.getMasterContext().getConf().getVar(ConfVars.SHUFFLE_FILE_FORMAT); for (int i = 0; i < scans.length; i++) {
public void assignToLeafTasks(LinkedList<TaskRequestEvent> taskRequests) throws InterruptedException { Collections.shuffle(taskRequests); LinkedList<TaskRequestEvent> remoteTaskRequests = new LinkedList<>(); String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext(). getConnectionInfo().getHostAndQMPort(); WorkerConnectionInfo connectionInfo = context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId()); if(connectionInfo == null) continue; int nodes = context.getMasterContext().getWorkerMap().size(); false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer(), queryMasterHostAndPort); context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
TableDesc tableDesc = masterContext.getTableDesc(scans[i]); for (Map.Entry<ExecutionBlockId, ExecutionBlockId> unionScanEntry: execBlock.getUnionScanMap().entrySet()) { ExecutionBlockId originScanEbId = unionScanEntry.getKey(); stats[i] += masterContext.getStage(originScanEbId).getResultStats().getNumBytes(); stats[i] = masterContext.getStage(scanEBId).getResultStats().getNumBytes(); StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); String intermediateDataFormat = schedulerContext.getMasterContext().getConf().getVar(ConfVars.SHUFFLE_FILE_FORMAT); for (int i = 0; i < scans.length; i++) {
CatalogService catalog = stage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalRootNode rootNode = masterPlan.getLogicalPlan().getRootBlock().getRoot(); TableDesc tableDesc = null; stage.getContext().getQueryContext(), tableDesc, sortNode.getInSchema(), List<ExecutionBlock> childBlocks = masterPlan.getChilds(stage.getId()); for (ExecutionBlock childBlock : childBlocks) { Stage childExecSM = stage.getContext().getStage(childBlock.getId()); for (Task qu : childExecSM.getTasks()) { for (IntermediateEntry p : qu.getIntermediateData()) {
scanEbId = childBlock.getId(); Stage childExecSM = stage.getContext().getStage(childBlock.getId()); TableDesc tableDesc = masterContext.getTableDesc(eachScan); Tablespace space = TablespaceManager.get(tableDesc.getUri());
scanEbId = childBlock.getId(); Stage childExecSM = stage.getContext().getStage(childBlock.getId()); TableDesc tableDesc = masterContext.getTableDesc(eachScan);
public void assignToLeafTasks(LinkedList<TaskRequestEvent> taskRequests) throws InterruptedException { Collections.shuffle(taskRequests); LinkedList<TaskRequestEvent> remoteTaskRequests = new LinkedList<TaskRequestEvent>(); String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext(). getConnectionInfo().getHostAndQMPort(); WorkerConnectionInfo connectionInfo = context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId()); if(connectionInfo == null) continue; int nodes = context.getMasterContext().getWorkerMap().size(); false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer(), queryMasterHostAndPort); context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
CatalogService catalog = stage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalRootNode rootNode = masterPlan.getLogicalPlan().getRootBlock().getRoot(); TableDesc tableDesc = null; stage.getContext().getQueryContext(), tableDesc, sortNode.getInSchema(), List<ExecutionBlock> childBlocks = masterPlan.getChilds(stage.getId()); for (ExecutionBlock childBlock : childBlocks) { Stage childExecSM = stage.getContext().getStage(childBlock.getId()); for (Task qu : childExecSM.getTasks()) { for (IntermediateEntry p : qu.getIntermediateData()) {
partitions.addAll(stage.getContext().getStage(block.getId()).getHashShuffleIntermediateEntries());
partitions.addAll(stage.getContext().getStage(block.getId()).getHashShuffleIntermediateEntries());
public void assignToNonLeafTasks(LinkedList<TaskRequestEvent> taskRequests) throws InterruptedException { Collections.shuffle(taskRequests); String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext(). getConnectionInfo().getHostAndQMPort(); false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer(), context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId()); context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
stage.getContext().getQueryMasterContext().getSingleEventExecutor() .submit(new Runnable() { @Override
public void assignToNonLeafTasks(LinkedList<TaskRequestEvent> taskRequests) throws InterruptedException { Collections.shuffle(taskRequests); String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext(). getConnectionInfo().getHostAndQMPort(); false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer(), context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId()); context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog, TablespaceManager.getInstance()); "", queryTaskContext.getEventHandler(), masterPlan); queryTaskContext.getEventHandler().handle(new QueryEvent(queryId, QueryEventType.START)); } catch (Throwable t) { LOG.error(t.getMessage(), t);
queryMasterTask.getQueryTaskContext().getDispatcher().start(); queryMasterTask.startQuery();
query.context.getQueryMasterContext().getWorkerContext(). getTaskHistoryWriter().appendHistory(writerFuture); } catch (Throwable e) {
stage.getContext().getQueryMasterContext().getSingleEventExecutor() .submit(new Runnable() { @Override