public ExecutableManager getExecutableManager() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); }
ExecutableManager getExecutableManger() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); } }
public boolean isEnableUHCDictStep() { if (!config.getConfig().isBuildUHCDictWithMREnabled()) { return false; } List<TblColRef> uhcColumns = seg.getCubeDesc().getAllUHCColumns(); if (uhcColumns.size() == 0) { return false; } return true; }
public boolean isEnableUHCDictStep() { if (!config.getConfig().isBuildUHCDictWithMREnabled()) { return false; } List<TblColRef> uhcColumns = seg.getCubeDesc().getAllUHCColumns(); if (uhcColumns.size() == 0) { return false; } return true; }
jobLock = lock; String serverMode = jobEngineConfig.getConfig().getServerMode(); if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) { logger.info("server mode: " + serverMode + ", no need to run job scheduler"); jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS, new SynchronousQueue<Runnable>()); context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());
@Override public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException { String serverMode = jobEngineConfig.getConfig().getServerMode(); if (!("job".equals(serverMode.toLowerCase(Locale.ROOT)) || "all".equals(serverMode.toLowerCase(Locale.ROOT)))) { logger.info("server mode: " + serverMode + ", no need to run job scheduler"); this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig()); jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS, new SynchronousQueue<Runnable>()); context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());
@Override public void run() { try (SetThreadName ignored = new SetThreadName("Scheduler %s Job %s", System.identityHashCode(DefaultScheduler.this), executable.getId())) { executable.execute(context); } catch (ExecuteException e) { logger.error("ExecuteException job:" + executable.getId(), e); } catch (Exception e) { if (AbstractExecutable.isMetaDataPersistException(e, 5)) { // Job fail due to PersistException ExecutableManager.getInstance(jobEngineConfig.getConfig()) .forceKillJobWithRetry(executable.getId()); } logger.error("unknown error execute job:" + executable.getId(), e); } finally { context.removeRunningJob(executable); } // trigger the next step asap fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } }
public SparkExecutable createFactDistinctColumnsSparkStep(String jobId) { final SparkExecutable sparkExecutable = new SparkExecutable(); final IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg); final String tablePath = JoinedFlatTable.getTableDir(flatTableDesc, getJobWorkingDir(jobId)); sparkExecutable.setClassName(SparkFactDistinct.class.getName()); sparkExecutable.setParam(SparkFactDistinct.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName()); sparkExecutable.setParam(SparkFactDistinct.OPTION_META_URL.getOpt(), getSegmentMetadataUrl(seg.getConfig(), jobId)); sparkExecutable.setParam(SparkFactDistinct.OPTION_INPUT_TABLE.getOpt(), seg.getConfig().getHiveDatabaseForIntermediateTable() + "." + flatTableDesc.getTableName()); sparkExecutable.setParam(SparkFactDistinct.OPTION_INPUT_PATH.getOpt(), tablePath); sparkExecutable.setParam(SparkFactDistinct.OPTION_OUTPUT_PATH.getOpt(), getFactDistinctColumnsPath(jobId)); sparkExecutable.setParam(SparkFactDistinct.OPTION_SEGMENT_ID.getOpt(), seg.getUuid()); sparkExecutable.setParam(SparkFactDistinct.OPTION_STATS_SAMPLING_PERCENT.getOpt(), String.valueOf(config.getConfig().getCubingInMemSamplingPercent())); sparkExecutable.setJobId(jobId); sparkExecutable.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS); sparkExecutable.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES, getCounterOuputPath(jobId)); StringBuilder jars = new StringBuilder(); StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars()); sparkExecutable.setJars(jars.toString()); return sparkExecutable; }
private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) { KylinConfig kylinConfig = config.getConfig(); CubeInstance cube = seg.getCubeInstance(); List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(), String msg = "Find more than one project containing the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! "; if (!config.getConfig().allowCubeAppearInMultipleProjects()) { throw new RuntimeException(msg); } else {
public MapReduceExecutable createCalculateStatsFromBaseCuboid(String inputPath, String outputPath, CuboidModeEnum cuboidMode) { MapReduceExecutable result = new MapReduceExecutable(); result.setName(ExecutableConstants.STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID); result.setMapReduceJobClass(CalculateStatsFromBaseCuboidJob.class); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_SAMPLING_PERCENT, String.valueOf(config.getConfig().getCubingInMemSamplingPercent())); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Calculate_Stats_For_Segment_" + seg.getRealization().getName() + "_Step"); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString()); result.setMapReduceParams(cmd.toString()); return result; }
public MapReduceExecutable createFactDistinctColumnsStep(String jobId) { MapReduceExecutable result = new MapReduceExecutable(); result.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS); result.setMapReduceJobClass(FactDistinctColumnsJob.class); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getFactDistinctColumnsPath(jobId)); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_OUTPUT, getStatisticsPath(jobId)); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_SAMPLING_PERCENT, String.valueOf(config.getConfig().getCubingInMemSamplingPercent())); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Fact_Distinct_Columns_" + seg.getRealization().getName() + "_Step"); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId); result.setMapReduceParams(cmd.toString()); result.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES); return result; }
private String schedulerId() { return ZOOKEEPER_LOCK_PATH + "/" + jobEngineConfig.getConfig().getMetadataUrlPrefix(); }
private String getZKConnectString(JobEngineConfig context) { Configuration conf = HadoopUtil.newHBaseConfiguration(context.getConfig().getStorageUrl()); return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT); }
ExecutableManager getExecutableManger() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); } }
public ExecutableManager getExecutableManager() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); }
public boolean isEnableUHCDictStep() { if (!config.getConfig().isBuildUHCDictWithMREnabled()) { return false; } List<TblColRef> uhcColumns = seg.getCubeDesc().getAllUHCColumns(); if (uhcColumns.size() == 0) { return false; } return true; }
public boolean isEnableUHCDictStep() { if (!config.getConfig().isBuildUHCDictWithMREnabled()) { return false; } List<TblColRef> uhcColumns = seg.getCubeDesc().getAllUHCColumns(); if (uhcColumns.size() == 0) { return false; } return true; }
protected String getIntermediateHiveTableName(IJoinedFlatTableDesc intermediateTableDesc, String jobUuid) { return engineConfig.getConfig().getHiveDatabaseForIntermediateTable() + "." + intermediateTableDesc.getTableName(jobUuid); }
public MapReduceExecutable createFactDistinctColumnsStep(String jobId) { MapReduceExecutable result = new MapReduceExecutable(); result.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS); result.setMapReduceJobClass(FactDistinctColumnsJob.class); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getFactDistinctColumnsPath(jobId)); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_OUTPUT, getStatisticsPath(jobId)); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_SAMPLING_PERCENT, String.valueOf(config.getConfig().getCubingInMemSamplingPercent())); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Fact_Distinct_Columns_" + seg.getRealization().getName() + "_Step"); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId); result.setMapReduceParams(cmd.toString()); result.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES); return result; }