@Override public JobConf createJobConf(final AddElementsFromHdfs operation, final String mapperGeneratorClassName, final Store store) throws IOException { final JobConf jobConf = new JobConf(new Configuration()); LOGGER.info("Setting up job conf"); jobConf.set(SCHEMA, new String(store.getSchema().toCompactJson(), CommonConstants.UTF_8)); LOGGER.debug("Added {} {} to job conf", SCHEMA, new String(store.getSchema().toCompactJson(), CommonConstants.UTF_8)); jobConf.set(MAPPER_GENERATOR, mapperGeneratorClassName); LOGGER.info("Added {} of {} to job conf", MAPPER_GENERATOR, mapperGeneratorClassName); jobConf.set(VALIDATE, String.valueOf(operation.isValidate())); LOGGER.info("Added {} option of {} to job conf", VALIDATE, operation.isValidate()); if (null != operation.getNumMapTasks()) { jobConf.setNumMapTasks(operation.getNumMapTasks()); LOGGER.info("Set number of map tasks to {} on job conf", operation.getNumMapTasks()); } if (null != operation.getNumReduceTasks()) { jobConf.setNumReduceTasks(operation.getNumReduceTasks()); LOGGER.info("Set number of reduce tasks to {} on job conf", operation.getNumReduceTasks()); } jobConf.set(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS, ((AccumuloStore) store).getKeyPackage().getKeyConverter().getClass().getName()); return jobConf; }
private void validateOperation(final AddElementsFromHdfs operation) { if (null != operation.getMinMapTasks()) { LOGGER.warn("minMapTasks field will be ignored"); } if (null != operation.getMaxMapTasks()) { LOGGER.warn("maxMapTasks field will be ignored"); } if (null != operation.getNumReduceTasks() && (null != operation.getMinReduceTasks() || null != operation.getMaxReduceTasks())) { throw new IllegalArgumentException("minReduceTasks and/or maxReduceTasks should not be set if numReduceTasks is"); } if (null != operation.getMinReduceTasks() && null != operation.getMaxReduceTasks()) { LOGGER.warn("Logic for the minimum may result in more reducers than the maximum set"); if (operation.getMinReduceTasks() > operation.getMaxReduceTasks()) { throw new IllegalArgumentException("Minimum number of reducers must be less than the maximum number of reducers"); } } if (null == operation.getSplitsFilePath()) { throw new IllegalArgumentException("splitsFilePath is required"); } if (null == operation.getWorkingPath()) { throw new IllegalArgumentException("workingPath is required"); } }
final int maxReducers; int numReducers; if (validateValue(operation.getNumReduceTasks()) != 0) { minReducers = validateValue(operation.getNumReduceTasks()); maxReducers = validateValue(operation.getNumReduceTasks()); } else { minReducers = validateValue(operation.getMinReduceTasks());