public static void ensurePaths(HadoopDruidIndexerConfig config) { authenticate(config); // config.addInputPaths() can have side-effects ( boo! :( ), so this stuff needs to be done before anything else try { Job job = Job.getInstance( new Configuration(), StringUtils.format("%s-determine_partitions-%s", config.getDataSource(), config.getIntervals()) ); job.getConfiguration().set("io.sort.record.percent", "0.19"); injectSystemProperties(job); config.addJobProperties(job); config.addInputPaths(job); } catch (IOException e) { throw Throwables.propagate(e); } }
job = Job.getInstance( new Configuration(), StringUtils.format("%s-index-generator-%s", config.getDataSource(), config.getIntervals()) );
groupByJob = Job.getInstance( new Configuration(), StringUtils.format("%s-determine_partitions_hashed-%s", config.getDataSource(), config.getIntervals()) );
groupByJob = Job.getInstance( new Configuration(), StringUtils.format("%s-determine_partitions_groupby-%s", config.getDataSource(), config.getIntervals()) ); StringUtils.format("%s-determine_partitions_dimselection-%s", config.getDataSource(), config.getIntervals()) );
public static void ensurePaths(HadoopDruidIndexerConfig config) { authenticate(config); // config.addInputPaths() can have side-effects ( boo! :( ), so this stuff needs to be done before anything else try { Job job = Job.getInstance( new Configuration(), StringUtils.format("%s-determine_partitions-%s", config.getDataSource(), config.getIntervals()) ); job.getConfiguration().set("io.sort.record.percent", "0.19"); injectSystemProperties(job); config.addJobProperties(job); config.addInputPaths(job); } catch (IOException e) { throw Throwables.propagate(e); } }
job = Job.getInstance( new Configuration(), StringUtils.format("%s-index-generator-%s", config.getDataSource(), config.getIntervals()) );
groupByJob = Job.getInstance( new Configuration(), StringUtils.format("%s-determine_partitions_groupby-%s", config.getDataSource(), config.getIntervals()) ); StringUtils.format("%s-determine_partitions_dimselection-%s", config.getDataSource(), config.getIntervals()) );
groupByJob = Job.getInstance( new Configuration(), StringUtils.format("%s-determine_partitions_hashed-%s", config.getDataSource(), config.getIntervals()) );