private static void addJoinShuffle(Stage stage, int partitionId, Map<ExecutionBlockId, List<IntermediateEntry>> grouppedPartitions) { Map<String, List<FetchProto>> fetches = new HashMap<>(); for (ExecutionBlock execBlock : stage.getMasterPlan().getChilds(stage.getId())) { if (grouppedPartitions.containsKey(execBlock.getId())) { String name = execBlock.getId().toString(); List<FetchProto> requests = mergeShuffleRequest(name, partitionId, HASH_SHUFFLE, grouppedPartitions.get(execBlock.getId())); fetches.put(name, requests); } } if (fetches.isEmpty()) { LOG.info(stage.getId() + "'s " + partitionId + " partition has empty result."); return; } Stage.scheduleFetches(stage, fetches); }
private static void addJoinShuffle(Stage stage, int partitionId, Map<ExecutionBlockId, List<IntermediateEntry>> grouppedPartitions) { Map<String, List<FetchProto>> fetches = new HashMap<>(); for (ExecutionBlock execBlock : stage.getMasterPlan().getChilds(stage.getId())) { if (grouppedPartitions.containsKey(execBlock.getId())) { String name = execBlock.getId().toString(); List<FetchProto> requests = mergeShuffleRequest(name, partitionId, HASH_SHUFFLE, grouppedPartitions.get(execBlock.getId())); fetches.put(name, requests); } } if (fetches.isEmpty()) { LOG.info(stage.getId() + "'s " + partitionId + " partition has empty result."); return; } Stage.scheduleFetches(stage, fetches); }
private static void schedule(Stage stage) throws IOException, TajoException { MasterPlan masterPlan = stage.getMasterPlan(); ExecutionBlock execBlock = stage.getBlock(); if (stage.getMasterPlan().isLeaf(execBlock.getId()) && execBlock.getScanNodes().length == 1) { // Case 1: Just Scan // Some execution blocks can have broadcast table even though they don't have any join nodes scheduleFragmentsForLeafQuery(stage); } else if (execBlock.getScanNodes().length > 1) { // Case 2: Join Repartitioner.scheduleFragmentsForJoinQuery(stage.schedulerContext, stage); } else { // Case 3: Others (Sort or Aggregation) int numTasks = getNonLeafTaskNum(stage); Repartitioner.scheduleFragmentsForNonLeafTasks(stage.schedulerContext, masterPlan, stage, numTasks); } }
private static void schedule(Stage stage) throws IOException, TajoException { MasterPlan masterPlan = stage.getMasterPlan(); ExecutionBlock execBlock = stage.getBlock(); if (stage.getMasterPlan().isLeaf(execBlock.getId()) && execBlock.getScanNodes().length == 1) { // Case 1: Just Scan // Some execution blocks can have broadcast table even though they don't have any join nodes scheduleFragmentsForLeafQuery(stage); } else if (execBlock.getScanNodes().length > 1) { // Case 2: Join Repartitioner.scheduleFragmentsForJoinQuery(stage.schedulerContext, stage); } else { // Case 3: Others (Sort or Aggregation) int numTasks = getNonLeafTaskNum(stage); Repartitioner.scheduleFragmentsForNonLeafTasks(stage.schedulerContext, masterPlan, stage, numTasks); } }
@Override public void start() { info(LOG, "Start TaskScheduler"); maximumRequestContainer = Math.min(tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_REQUEST_MAX_NUM) , stage.getContext().getWorkerMap().size()); if (isLeaf) { candidateWorkers.addAll(getWorkerIds(getLeafTaskHosts())); } else { //find assigned hosts for Non-Leaf locality in children executionBlock List<ExecutionBlock> executionBlockList = stage.getMasterPlan().getChilds(stage.getBlock()); for (ExecutionBlock executionBlock : executionBlockList) { Stage childStage = stage.getContext().getStage(executionBlock.getId()); candidateWorkers.addAll(childStage.getAssignedWorkerMap().keySet()); } } this.schedulingThread.start(); super.start(); }
@Override public void start() { info(LOG, "Start TaskScheduler"); maximumRequestContainer = Math.min(tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_REQUEST_MAX_NUM) , stage.getContext().getWorkerMap().size()); if (isLeaf) { candidateWorkers.addAll(getWorkerIds(getLeafTaskHosts())); } else { //find assigned hosts for Non-Leaf locality in children executionBlock List<ExecutionBlock> executionBlockList = stage.getMasterPlan().getChilds(stage.getBlock()); for (ExecutionBlock executionBlock : executionBlockList) { Stage childStage = stage.getContext().getStage(executionBlock.getId()); candidateWorkers.addAll(childStage.getAssignedWorkerMap().keySet()); } } this.schedulingThread.start(); super.start(); }
private void handleQueryFailure(Query query, Stage lastStage) { QueryContext context = query.context.getQueryContext(); if (lastStage != null && context.hasOutputTableUri()) { Tablespace space = TablespaceManager.get(context.getOutputTableUri()); try { LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); space.rollbackTable(rootNode.getChild()); } catch (Throwable e) { LOG.warn(query.getId() + ", failed processing cleanup storage when query failed:" + e.getMessage(), e); } } }
private void handleQueryFailure(Query query, Stage lastStage) { QueryContext context = query.context.getQueryContext(); if (lastStage != null && context.hasOutputTableUri()) { Tablespace space = TablespaceManager.get(context.getOutputTableUri()); try { LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); space.rollbackTable(rootNode.getChild()); } catch (Throwable e) { LOG.warn(query.getId() + ", failed processing cleanup storage when query failed:" + e.getMessage(), e); } } }
private void initTaskScheduler(Stage stage) throws IOException { TajoConf conf = stage.context.getConf(); stage.schedulerContext = new TaskSchedulerContext(stage.context, stage.getMasterPlan().isLeaf(stage.getId()), stage.getId()); stage.taskScheduler = TaskSchedulerFactory.get(conf, stage.schedulerContext, stage); stage.taskScheduler.init(conf); LOG.info(stage.taskScheduler.getName() + " is chosen for the task scheduling for " + stage.getId()); }
private void initTaskScheduler(Stage stage) throws IOException { TajoConf conf = stage.context.getConf(); stage.schedulerContext = new TaskSchedulerContext(stage.context, stage.getMasterPlan().isLeaf(stage.getId()), stage.getId()); stage.taskScheduler = TaskSchedulerFactory.get(conf, stage.schedulerContext, stage); stage.taskScheduler.init(conf); LOG.info(stage.taskScheduler.getName() + " is chosen for the task scheduling for " + stage.getId()); }
/** * Sends stopping request to all worker */ protected void stopExecutionBlock() { // If there are still live tasks, try to kill the tasks. and send the shuffle report request List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList(); if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) { List<ExecutionBlock> childs = getMasterPlan().getChilds(getId()); for (ExecutionBlock executionBlock : childs) { ebIds.add(executionBlock.getId().getProto()); } } StopExecutionBlockRequest.Builder stopRequest = StopExecutionBlockRequest.newBuilder(); ExecutionBlockListProto.Builder cleanupList = ExecutionBlockListProto.newBuilder(); cleanupList.addAllExecutionBlockId(Lists.newArrayList(ebIds)); stopRequest.setCleanupList(cleanupList.build()); stopRequest.setExecutionBlockId(getId().getProto()); sendStopExecutionBlockEvent(stopRequest.build()); }
/** * Sends stopping request to all worker */ protected void stopExecutionBlock() { // If there are still live tasks, try to kill the tasks. and send the shuffle report request List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList(); if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) { List<ExecutionBlock> childs = getMasterPlan().getChilds(getId()); for (ExecutionBlock executionBlock : childs) { ebIds.add(executionBlock.getId().getProto()); } } StopExecutionBlockRequest.Builder stopRequest = StopExecutionBlockRequest.newBuilder(); ExecutionBlockListProto.Builder cleanupList = ExecutionBlockListProto.newBuilder(); cleanupList.addAllExecutionBlockId(Lists.newArrayList(ebIds)); stopRequest.setCleanupList(cleanupList.build()); stopRequest.setExecutionBlockId(getId().getProto()); sendStopExecutionBlockEvent(stopRequest.build()); }
long volume = getInputVolume(stage.getMasterPlan(), stage.context, stage.getBlock());
minTaskMemory = tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY); schedulerDelay= tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_DELAY); isLeaf = stage.getMasterPlan().isLeaf(stage.getBlock());
minTaskMemory = tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY); schedulerDelay= tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_DELAY); isLeaf = stage.getMasterPlan().isLeaf(stage.getBlock());
LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); CatalogService catalog = lastStage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); query.context.getQueryContext(), lastStage.getId(), lastStage.getMasterPlan().getLogicalPlan(), lastStage.getOutSchema(), tableDesc);
List<ColumnStats> columnStatses = StatisticsUtil.emptyColumnStats(stage.getDataChannel().getSchema()); MasterPlan masterPlan = stage.getMasterPlan(); for (ExecutionBlock block : masterPlan.getChilds(stage.getBlock())) { Stage childStage = stage.context.getStage(block.getId());
List<ColumnStats> columnStatses = StatisticsUtil.emptyColumnStats(stage.getDataChannel().getSchema()); MasterPlan masterPlan = stage.getMasterPlan(); for (ExecutionBlock block : masterPlan.getChilds(stage.getBlock())) { Stage childStage = stage.context.getStage(block.getId());
MasterPlan masterPlan = stage.getMasterPlan(); keys = channel.getShuffleKeys(); if (!masterPlan.isRoot(stage.getBlock()) ) {
MasterPlan masterPlan = stage.getMasterPlan(); keys = channel.getShuffleKeys(); if (!masterPlan.isRoot(stage.getBlock()) ) {