private void validateOperation(final AddElementsFromHdfs operation) { if (null != operation.getMinMapTasks()) { LOGGER.warn("minMapTasks field will be ignored"); } if (null != operation.getMaxMapTasks()) { LOGGER.warn("maxMapTasks field will be ignored"); } if (null != operation.getNumReduceTasks() && (null != operation.getMinReduceTasks() || null != operation.getMaxReduceTasks())) { throw new IllegalArgumentException("minReduceTasks and/or maxReduceTasks should not be set if numReduceTasks is"); } if (null != operation.getMinReduceTasks() && null != operation.getMaxReduceTasks()) { LOGGER.warn("Logic for the minimum may result in more reducers than the maximum set"); if (operation.getMinReduceTasks() > operation.getMaxReduceTasks()) { throw new IllegalArgumentException("Minimum number of reducers must be less than the maximum number of reducers"); } } if (null == operation.getSplitsFilePath()) { throw new IllegalArgumentException("splitsFilePath is required"); } if (null == operation.getWorkingPath()) { throw new IllegalArgumentException("workingPath is required"); } }
protected void setUpPartitionerFromUserProvidedSplitsFile(final Job job, final AddElementsFromHdfs operation) throws IOException { final String splitsFilePath = operation.getSplitsFilePath(); if (validateValue(operation.getMaxReduceTasks()) != -1 || validateValue(operation.getMinReduceTasks()) != -1) { LOGGER.info("Using splits file provided by user {}, ignoring minReduceTasks and maxReduceTasks", splitsFilePath); } else { LOGGER.info("Using splits file provided by user {}", splitsFilePath); } final int numSplits = IngestUtils.getNumSplits(FileSystem.get(job.getConfiguration()), new Path(splitsFilePath)); job.setNumReduceTasks(numSplits + 1); job.setPartitionerClass(GafferKeyRangePartitioner.class); GafferKeyRangePartitioner.setSplitFile(job, splitsFilePath); }
maxReducers = validateValue(operation.getNumReduceTasks()); } else { minReducers = validateValue(operation.getMinReduceTasks()); maxReducers = validateValue(operation.getMaxReduceTasks());