public static IMRBatchCubingInputSide getBatchCubingInputSide(CubeSegment seg) { IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(seg); return (IMRBatchCubingInputSide)SourceManager.createEngineAdapter(seg, IMRInput.class).getBatchCubingInputSide(flatDesc); }
private void submitJob(CubeInstance cube, TSRange tsRange, CubeBuildTypeEnum buildType, boolean forceMergeEmptySeg, String submitter) throws IOException, JobException { checkCubeDescSignature(cube); DefaultChainedExecutable job; if (buildType == CubeBuildTypeEnum.BUILD) { CubeSegment newSeg = cubeManager.appendSegment(cube, tsRange); job = EngineFactory.createBatchCubingJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.MERGE) { CubeSegment newSeg = cubeManager.mergeSegments(cube, tsRange, null, forceMergeEmptySeg); job = EngineFactory.createBatchMergeJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.REFRESH) { CubeSegment refreshSeg = cubeManager.refreshSegment(cube, tsRange, null); job = EngineFactory.createBatchCubingJob(refreshSeg, submitter); } else { throw new JobException("invalid build type:" + buildType); } executableManager.addJob(job); }
/** Optimize a segment based on the cuboid recommend list produced by the cube planner. */ public static DefaultChainedExecutable createBatchOptimizeJob(CubeSegment optimizeSegment, String submitter) { return batchEngine(optimizeSegment).createBatchOptimizeJob(optimizeSegment, submitter); } }
private void mergeCubeSegment(String cubeName) { CubeInstance cube = getCubeManager().getCube(cubeName); if (!cube.needAutoMerge()) return; synchronized (CubeService.class) { try { cube = getCubeManager().getCube(cubeName); SegmentRange offsets = cube.autoMergeCubeSegments(); if (offsets != null) { CubeSegment newSeg = getCubeManager().mergeSegments(cube, null, offsets, true); logger.debug("Will submit merge job on " + newSeg); DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM"); getExecutableManager().addJob(job); } else { logger.debug("Not ready for merge on cube " + cubeName); } } catch (IOException e) { logger.error("Failed to auto merge cube " + cubeName, e); } } }
DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter); getExecutableManager().addJob(optimizeJob);
DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
public static ISparkBatchCubingInputSide getBatchCubingInputSide(CubeSegment seg) { IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(seg); return (ISparkBatchCubingInputSide)SourceManager.createEngineAdapter(seg, ISparkInput.class).getBatchCubingInputSide(flatDesc); }
src = source.enrichSourcePartitionBeforeBuild(cube, src); newSeg = getCubeManager().appendSegment(cube, src); job = EngineFactory.createBatchCubingJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.MERGE) { newSeg = getCubeManager().mergeSegments(cube, tsRange, segRange, force); job = EngineFactory.createBatchMergeJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.REFRESH) { newSeg = getCubeManager().refreshSegment(cube, tsRange, segRange); job = EngineFactory.createBatchCubingJob(newSeg, submitter); } else { throw new BadRequestException(String.format(Locale.ROOT, msg.getINVALID_BUILD_TYPE(), buildType));
/** Build a new cube segment, typically its time range appends to the end of current cube. */ public static DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) { return batchEngine(newSegment).createBatchCubingJob(newSegment, submitter); }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID); final KylinConfig kylinConfig = AbstractHadoopJob.loadKylinPropsAndMetadata(); cube = CubeManager.getInstance(kylinConfig).getCube(cubeName); cubeDesc = cube.getDescriptor(); cubeSegment = cube.getSegmentById(segmentID); CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); Map<TblColRef, Dictionary<String>> dictionaryMap = DictionaryGetterUtil.getDictionaryMap(cubeSegment, context.getInputSplit(), context.getConfiguration()); baseCuboidBuilder = new BaseCuboidBuilder(kylinConfig, cubeDesc, cubeSegment, intermediateTableDesc, dictionaryMap); }
private void submitJob(CubeInstance cube, TSRange tsRange, CubeBuildTypeEnum buildType, boolean forceMergeEmptySeg, String submitter) throws IOException, JobException { checkCubeDescSignature(cube); DefaultChainedExecutable job; if (buildType == CubeBuildTypeEnum.BUILD) { CubeSegment newSeg = cubeManager.appendSegment(cube, tsRange); job = EngineFactory.createBatchCubingJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.MERGE) { CubeSegment newSeg = cubeManager.mergeSegments(cube, tsRange, null, forceMergeEmptySeg); job = EngineFactory.createBatchMergeJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.REFRESH) { CubeSegment refreshSeg = cubeManager.refreshSegment(cube, tsRange, null); job = EngineFactory.createBatchCubingJob(refreshSeg, submitter); } else { throw new JobException("invalid build type:" + buildType); } executableManager.addJob(job); }
/** Merge multiple small segments into a big one. */ public static DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter) { return batchEngine(mergeSegment).createBatchMergeJob(mergeSegment, submitter); }
@Override protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); config = AbstractHadoopJob.loadKylinPropsAndMetadata(); cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); cube = CubeManager.getInstance(config).getCube(cubeName); cubeDesc = cube.getDescriptor(); cubeSeg = cube.getSegmentById(conf.get(BatchConstants.CFG_CUBE_SEGMENT_ID)); flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat(); intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSeg), cubeDesc); globalColumns = cubeDesc.getAllGlobalDictColumns(); globalColumnIndex = new int[globalColumns.size()]; globalColumnValues = Lists.newArrayListWithExpectedSize(globalColumns.size()); for (int i = 0; i < globalColumns.size(); i++) { TblColRef colRef = globalColumns.get(i); int columnIndexOnFlatTbl = intermediateTableDesc.getColumnIndex(colRef); globalColumnIndex[i] = columnIndexOnFlatTbl; globalColumnValues.add(Sets.<String> newHashSet()); } splitKey = DictionaryGetterUtil.getInputSplitSignature(cubeSeg, context.getInputSplit()); }
/** Mark deprecated to indicate for test purpose only */ @Deprecated public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) { return batchEngine(cubeDesc).getJoinedFlatTableDesc(cubeDesc); }
@Override protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); cube = CubeManager.getInstance(config).getCube(cubeName); cubeSeg = cube.getSegmentById(conf.get(BatchConstants.CFG_CUBE_SEGMENT_ID)); cubeDesc = cube.getDescriptor(); baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); reducerMapping = new FactDistinctColumnsReducerMapping(cube); allCols = reducerMapping.getAllDimDictCols(); flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat(); intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSeg), cubeDesc); columnIndex = new int[allCols.size()]; for (int i = 0; i < allCols.size(); i++) { TblColRef colRef = allCols.get(i); int columnIndexOnFlatTbl = intermediateTableDesc.getColumnIndex(colRef); columnIndex[i] = columnIndexOnFlatTbl; } }
public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) { return batchEngine(newSegment).getJoinedFlatTableDesc(newSegment); }
String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID); cubeSegment = cube.getSegmentById(segmentID); flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
/** Build a new cube segment, typically its time range appends to the end of current cube. */ public static DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) { return batchEngine(newSegment).createBatchCubingJob(newSegment, submitter); }
public void configureSparkJob(final CubeSegment seg, final SparkExecutable sparkExecutable, final String jobId, final String cuboidRootPath) { final IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg); final String tablePath = JoinedFlatTable.getTableDir(flatTableDesc, getJobWorkingDir(jobId)); sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName()); sparkExecutable.setParam(SparkCubingByLayer.OPTION_SEGMENT_ID.getOpt(), seg.getUuid()); sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_TABLE.getOpt(), seg.getConfig().getHiveDatabaseForIntermediateTable() + "." + flatTableDesc.getTableName()); sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_PATH.getOpt(), tablePath); sparkExecutable.setParam(SparkCubingByLayer.OPTION_META_URL.getOpt(), getSegmentMetadataUrl(seg.getConfig(), jobId)); sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath); sparkExecutable.setJobId(jobId); StringBuilder jars = new StringBuilder(); StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars()); sparkExecutable.setJars(jars.toString()); sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_SPARK_CUBE); }
/** Merge multiple small segments into a big one. */ public static DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter) { return batchEngine(mergeSegment).createBatchMergeJob(mergeSegment, submitter); }