public boolean isNumReduceTasksSetByUser() { return (groupingOptions != null && groupingOptions.getNumReducers() > 0); } }
public void configureShuffle(Job job) { ptype.configureShuffle(job, groupingOptions); if (groupingOptions == null || groupingOptions.getNumReducers() <= 0) { long bytesPerTask = job.getConfiguration().getLong("crunch.bytes.per.reduce.task", (1000L * 1000L * 1000L)); int numReduceTasks = 1 + (int) (getSize() / bytesPerTask); if (numReduceTasks > 0) { job.setNumReduceTasks(numReduceTasks); LOG.info(String.format("Setting num reduce tasks to %d", numReduceTasks)); } else { LOG.warn("Attempted to set a negative number of reduce tasks"); } } }
public void configureShuffle(Job job) { ptype.configureShuffle(job, groupingOptions); if (groupingOptions == null || groupingOptions.getNumReducers() <= 0) { long bytesPerTask = job.getConfiguration().getLong("crunch.bytes.per.reduce.task", (1000L * 1000L * 1000L)); int numReduceTasks = 1 + (int) (getSize() / bytesPerTask); if (numReduceTasks > 0) { job.setNumReduceTasks(numReduceTasks); LOG.info(String.format("Setting num reduce tasks to %d", numReduceTasks)); } else { LOG.warn("Attempted to set a negative number of reduce tasks"); } } }
public int getNumReduceTasks() { int numReduceTasks; if (groupingOptions == null || groupingOptions.getNumReducers() <= 0) { numReduceTasks = PartitionUtils.getRecommendedPartitions(this, getPipeline().getConfiguration()); } else { numReduceTasks = groupingOptions.getNumReducers(); } return numReduceTasks; }
public void configureShuffle(Job job) { ptype.configureShuffle(job, groupingOptions); if (groupingOptions == null || groupingOptions.getNumReducers() <= 0) { int numReduceTasks = PartitionUtils.getRecommendedPartitions(this, getPipeline().getConfiguration()); if (numReduceTasks > 0) { job.setNumReduceTasks(numReduceTasks); LOG.info("Setting num reduce tasks to {}", numReduceTasks); } else { LOG.warn("Attempted to set a negative number of reduce tasks"); } } }
int numPartitions = (groupingOptions.getNumReducers() > 0) ? groupingOptions.getNumReducers() : PartitionUtils.getRecommendedPartitions(this, getPipeline().getConfiguration()); if (numPartitions <= 0) {
int numPartitions = (groupingOptions.getNumReducers() > 0) ? groupingOptions.getNumReducers() : PartitionUtils.getRecommendedPartitions(this, getPipeline().getConfiguration()); if (numPartitions <= 0) {