/** * * @param jobType job config file suffix name; if be null, will use the default job conf * @return the job config file path * @throws IOException */ public String getHadoopJobConfFilePath(String jobType) throws IOException { String suffix = null; if (!StringUtils.isEmpty(jobType)) { suffix = jobType; } String path = getHadoopJobConfFilePath(suffix, true); if (StringUtils.isEmpty(path)) { path = getHadoopJobConfFilePath(jobType, true); if (StringUtils.isEmpty(path)) { path = getHadoopJobConfFilePath(jobType, false); if (StringUtils.isEmpty(path)) { path = ""; } } } return path; }
public HBaseLookupMRSteps(CubeInstance cube) { this.cube = cube; this.config = new JobEngineConfig(cube.getConfig()); }
public static String getJobWorkingDir(JobEngineConfig conf, String jobId) { return getJobWorkingDir(conf.getHdfsWorkingDirectory(), jobId); }
@Test public void testPropertiesHotLoad() throws IOException { KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); JobEngineConfig jobEngineConfig = new JobEngineConfig(baseConfig); assertEquals(10, jobEngineConfig.getMaxConcurrentJobLimit()); updateProperty("kylin.job.max-concurrent-jobs", "20"); KylinConfig.getInstanceFromEnv().reloadFromSiteProperties(); assertEquals(20, jobEngineConfig.getMaxConcurrentJobLimit()); } }
ExecutableManager getExecutableManger() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); } }
protected boolean isJobPoolFull() { Map<String, Executable> runningJobs = context.getRunningJobs(); if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) { logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time"); return true; } return false; }
private String getHadoopJobConfFilePath(String suffix, boolean appendSuffix) throws IOException { String hadoopJobConfFile; if (suffix != null && appendSuffix) { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + "_" + suffix.toLowerCase(Locale.ROOT) + ".xml"); } else { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + ".xml"); } File jobConfig = getJobConfig(hadoopJobConfFile); if (jobConfig == null || !jobConfig.exists()) { logger.warn("fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml"); if (jobConfig == null || !jobConfig.exists()) { logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); throw new RuntimeException("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); } } return OptionsHelper.convertToFileURL(jobConfig.getAbsolutePath()); }
public JobBuilderSupport(CubeSegment seg, String submitter) { Preconditions.checkNotNull(seg, "segment cannot be null"); this.config = new JobEngineConfig(seg.getConfig()); this.seg = seg; this.submitter = submitter; }
public ExecutableManager getExecutableManager() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); }
public void appendMapReduceParameters(StringBuilder buf, String jobType) { try { String jobConf = config.getHadoopJobConfFilePath(jobType); if (jobConf != null && jobConf.length() > 0) { buf.append(" -conf ").append(jobConf); } } catch (IOException e) { throw new RuntimeException(e); } }
public BaseBatchCubingInputSide(CubeSegment seg, IJoinedFlatTableDesc flatDesc) { this.conf = new JobEngineConfig(KylinConfig.getInstanceFromEnv()); this.config = seg.getConfig(); this.flatDesc = flatDesc; this.hiveTableDatabase = config.getHiveDatabaseForIntermediateTable(); this.seg = seg; this.cubeDesc = seg.getCubeDesc(); this.cubeName = seg.getCubeInstance().getName(); }
public boolean isEnableUHCDictStep() { if (!config.getConfig().isBuildUHCDictWithMREnabled()) { return false; } List<TblColRef> uhcColumns = seg.getCubeDesc().getAllUHCColumns(); if (uhcColumns.size() == 0) { return false; } return true; }
public void appendMapReduceParameters(StringBuilder buf, String jobType) { try { String jobConf = config.getHadoopJobConfFilePath(jobType); if (jobConf != null && jobConf.length() > 0) { buf.append(" -conf ").append(jobConf); } } catch (IOException e) { throw new RuntimeException(e); } }
@SuppressWarnings("unchecked") @Override public void afterPropertiesSet() throws Exception { String timeZone = getConfig().getTimeZone(); TimeZone tzone = TimeZone.getTimeZone(timeZone); TimeZone.setDefault(tzone); final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); final Scheduler<AbstractExecutable> scheduler = (Scheduler<AbstractExecutable>) SchedulerFactory .scheduler(kylinConfig.getSchedulerType()); scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock()); Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { try { scheduler.shutdown(); } catch (SchedulerException e) { logger.error("error occurred to shutdown scheduler", e); } } })); }
public boolean isEnableUHCDictStep() { if (!config.getConfig().isBuildUHCDictWithMREnabled()) { return false; } List<TblColRef> uhcColumns = seg.getCubeDesc().getAllUHCColumns(); if (uhcColumns.size() == 0) { return false; } return true; }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { try { config = new JobEngineConfig(context.getConfig()); List<String> toDeletePaths = getDeletePaths(); dropHdfsPathOnCluster(toDeletePaths, HadoopUtil.getWorkingFileSystem()); if (StringUtils.isNotEmpty(context.getConfig().getHBaseClusterFs())) { dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration())); } } catch (IOException e) { logger.error("job:" + getId() + " execute finished with exception", e); output.append("\n").append(e.getLocalizedMessage()); return new ExecuteResult(ExecuteResult.State.ERROR, output.toString(), e); } return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString()); }
@Override public void run() { try (SetThreadName ignored = new SetThreadName("Scheduler %s Job %s", System.identityHashCode(DefaultScheduler.this), executable.getId())) { executable.execute(context); } catch (ExecuteException e) { logger.error("ExecuteException job:" + executable.getId(), e); } catch (Exception e) { if (AbstractExecutable.isMetaDataPersistException(e, 5)) { // Job fail due to PersistException ExecutableManager.getInstance(jobEngineConfig.getConfig()) .forceKillJobWithRetry(executable.getId()); } logger.error("unknown error execute job:" + executable.getId(), e); } finally { context.removeRunningJob(executable); } // trigger the next step asap fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } }
protected void startScheduler() throws SchedulerException { scheduler = DefaultScheduler.createInstance(); scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock()); if (!scheduler.hasStarted()) { throw new RuntimeException("scheduler has not been started"); } }
public MapReduceExecutable createFactDistinctColumnsStep(String jobId) { MapReduceExecutable result = new MapReduceExecutable(); result.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS); result.setMapReduceJobClass(FactDistinctColumnsJob.class); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getFactDistinctColumnsPath(jobId)); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_OUTPUT, getStatisticsPath(jobId)); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_SAMPLING_PERCENT, String.valueOf(config.getConfig().getCubingInMemSamplingPercent())); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Fact_Distinct_Columns_" + seg.getRealization().getName() + "_Step"); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId); result.setMapReduceParams(cmd.toString()); result.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES); return result; }
public MapReduceExecutable createCalculateStatsFromBaseCuboid(String inputPath, String outputPath, CuboidModeEnum cuboidMode) { MapReduceExecutable result = new MapReduceExecutable(); result.setName(ExecutableConstants.STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID); result.setMapReduceJobClass(CalculateStatsFromBaseCuboidJob.class); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName()); appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid()); appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, inputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath); appendExecCmdParameters(cmd, BatchConstants.ARG_STATS_SAMPLING_PERCENT, String.valueOf(config.getConfig().getCubingInMemSamplingPercent())); appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Calculate_Stats_For_Segment_" + seg.getRealization().getName() + "_Step"); appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString()); result.setMapReduceParams(cmd.toString()); return result; }