public void setShardSpecs(Map<Long, List<HadoopyShardSpec>> shardSpecs) { this.schema = schema.withTuningConfig(schema.getTuningConfig().withShardSpecs(shardSpecs)); this.pathSpec = JSON_MAPPER.convertValue(schema.getIOConfig().getPathSpec(), PathSpec.class); }
final HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSpec( theSchema .withTuningConfig(theSchema.getTuningConfig().withVersion(version)) );
public String runTask(String[] args) throws Exception { final String schema = args[0]; final String workingPath = args[1]; final String segmentOutputPath = args[2]; final String hadoopJobIdFile = args[3]; final HadoopIngestionSpec theSchema = HadoopDruidIndexerConfig.JSON_MAPPER .readValue( schema, HadoopIngestionSpec.class ); final HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSpec( theSchema .withIOConfig(theSchema.getIOConfig().withSegmentOutputPath(segmentOutputPath)) .withTuningConfig(theSchema.getTuningConfig().withWorkingPath(workingPath)) ); job = new HadoopDruidDetermineConfigurationJob(config); job.setHadoopJobIdFile(hadoopJobIdFile); log.info("Starting a hadoop determine configuration job..."); if (job.run()) { return HadoopDruidIndexerConfig.JSON_MAPPER.writeValueAsString( new HadoopDetermineConfigInnerProcessingStatus(config.getSchema(), job.getStats(), null) ); } else { return HadoopDruidIndexerConfig.JSON_MAPPER.writeValueAsString( new HadoopDetermineConfigInnerProcessingStatus(null, job.getStats(), job.getErrorMessage()) ); } }
public void setShardSpecs(Map<Long, List<HadoopyShardSpec>> shardSpecs) { this.schema = schema.withTuningConfig(schema.getTuningConfig().withShardSpecs(shardSpecs)); this.pathSpec = JSON_MAPPER.convertValue(schema.getIOConfig().getPathSpec(), PathSpec.class); }
final HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSpec( theSchema .withTuningConfig(theSchema.getTuningConfig().withVersion(version)) );
public String runTask(String[] args) throws Exception { final String schema = args[0]; final String workingPath = args[1]; final String segmentOutputPath = args[2]; final HadoopIngestionSpec theSchema = HadoopDruidIndexerConfig.JSON_MAPPER .readValue( schema, HadoopIngestionSpec.class ); final HadoopDruidIndexerConfig config = HadoopDruidIndexerConfig.fromSpec( theSchema .withIOConfig(theSchema.getIOConfig().withSegmentOutputPath(segmentOutputPath)) .withTuningConfig(theSchema.getTuningConfig().withWorkingPath(workingPath)) ); job = new HadoopDruidDetermineConfigurationJob(config); log.info("Starting a hadoop determine configuration job..."); if (job.run()) { return HadoopDruidIndexerConfig.JSON_MAPPER.writeValueAsString( new HadoopDetermineConfigInnerProcessingStatus(config.getSchema(), job.getStats(), null) ); } else { return HadoopDruidIndexerConfig.JSON_MAPPER.writeValueAsString( new HadoopDetermineConfigInnerProcessingStatus(null, job.getStats(), job.getErrorMessage()) ); } }