private Configuration getTableConf() { return tableEnv.getConfig().getConf(); }
private void setDefaultRes(NodeResource resource) { double cpu = NodeResourceUtil.getDefaultCpu(tEnv.getConfig().getConf()); int heap = NodeResourceUtil.getDefaultHeapMem(tEnv.getConfig().getConf()); int direct = NodeResourceUtil.getDefaultDirectMem(tEnv.getConfig().getConf()); resource.setCpu(cpu); resource.setHeapMem(heap); resource.setDirectMem(direct); }
@Override public List<ExecNode<?, ?>> process(List<ExecNode<?, ?>> sinkNodes, DAGProcessContext context) { sinkNodes.forEach(s -> Preconditions.checkArgument(s instanceof BatchExecNode)); tEnv = context.getTableEnvironment(); NodeResourceUtil.InferMode inferMode = NodeResourceUtil.getInferMode(tEnv.getConfig().getConf()); BatchExecNodeVisitor managedVisitor; if (inferMode.equals(NodeResourceUtil.InferMode.ALL)) { managedVisitor = new BatchManagedMemCalculatorOnStatistics(tEnv.getConfig().getConf()); } else { managedVisitor = new BatchManagedMemCalculatorOnConfig(tEnv.getConfig().getConf()); } sinkNodes.forEach(s -> ((BatchExecNode) s).accept(managedVisitor)); return sinkNodes; } }
private void calculateTableSourceScan(CommonScan tableSourceScan) { // user may have set resource for source transformation. StreamTransformation transformation = tableSourceScan.getSourceTransformation(tEnv.execEnv()); ResourceSpec sourceRes = transformation.getMinResources(); if (sourceRes == ResourceSpec.DEFAULT || sourceRes == null) { int heap = NodeResourceUtil.getSourceMem(tEnv.getConfig().getConf()); int direct = NodeResourceUtil.getSourceDirectMem(tEnv.getConfig().getConf()); sourceRes = NodeResourceUtil.getResourceSpec(tEnv.getConfig().getConf(), heap, direct); } calculateCommonScan(tableSourceScan, sourceRes); }
private void calculateCommonScan(CommonScan commonScan, ResourceSpec sourceRes) { ResourceSpec conversionRes = ResourceSpec.DEFAULT; if (commonScan.needInternalConversion()) { conversionRes = NodeResourceUtil.getDefaultResourceSpec(tEnv.getConfig().getConf()); } commonScan.setResForSourceAndConversion(sourceRes, conversionRes); }
protected void calculateOnShuffleStages(Map<ExecNode<?, ?>, ShuffleStage> nodeShuffleStageMap, DAGProcessContext context) { Configuration tableConf = tEnv.getConfig().getConf(); NodeResourceUtil.InferMode inferMode = NodeResourceUtil.getInferMode(tableConf); getShuffleStageParallelismCalculator(tableConf, inferMode).calculate(nodeShuffleStageMap.values()); Double cpuLimit = tableConf.getDouble(TableConfigOptions.SQL_RESOURCE_RUNNING_UNIT_CPU_TOTAL); if (cpuLimit > 0) { Map<BatchExecNode<?>, Set<NodeRunningUnit>> nodeRunningUnitMap = context.getRunningUnitMap(); BatchParallelismAdjuster.adjustParallelism(cpuLimit, nodeRunningUnitMap, nodeShuffleStageMap); } }
private void calculateDefault(ExecNode<?, ?> execNode) { execNode.getResource().setParallelism(NodeResourceUtil.getOperatorDefaultParallelism( tEnv.getConfig().getConf(), tEnv.execEnv().getParallelism())); }
private void calculateTableSourceScan(StreamExecTableSourceScan tableSourceScan) { StreamTransformation transformation = tableSourceScan.getSourceTransformation(tEnv.execEnv()); if (transformation.getMaxParallelism() > 0) { tableSourceScan.getResource().setParallelism(transformation.getMaxParallelism()); return; } int configParallelism = NodeResourceUtil.getSourceParallelism( tEnv.getConfig().getConf(), tEnv.execEnv().getParallelism()); tableSourceScan.getResource().setParallelism(configParallelism); }