private void setSplitSize(JobContext cx) { super.setMaxSplitSize(cx.getConfiguration().getLong(COMPACTION_JOB_MAPRED_MAX_SPLIT_SIZE, DEFAULT_COMPACTION_JOB_MAPRED_MAX_SPLIT_SIZE)); super.setMinSplitSizeNode(cx.getConfiguration().getLong(COMPACTION_JOB_MAPRED_MIN_SPLIT_SIZE, DEFAULT_COMPACTION_JOB_MAPRED_MIN_SPLIT_SIZE)); }
private void setSplitSize(JobContext cx) { super.setMaxSplitSize(cx.getConfiguration().getLong(COMPACTION_JOB_MAPRED_MAX_SPLIT_SIZE, DEFAULT_COMPACTION_JOB_MAPRED_MAX_SPLIT_SIZE)); super.setMinSplitSizeNode(cx.getConfiguration().getLong(COMPACTION_JOB_MAPRED_MIN_SPLIT_SIZE, DEFAULT_COMPACTION_JOB_MAPRED_MIN_SPLIT_SIZE)); }
private void setSplitSize(JobContext cx) { super.setMaxSplitSize(cx.getConfiguration().getLong(COMPACTION_JOB_MAPRED_MAX_SPLIT_SIZE, DEFAULT_COMPACTION_JOB_MAPRED_MAX_SPLIT_SIZE)); super.setMinSplitSizeNode(cx.getConfiguration().getLong(COMPACTION_JOB_MAPRED_MIN_SPLIT_SIZE, DEFAULT_COMPACTION_JOB_MAPRED_MIN_SPLIT_SIZE)); }
@Override public List<InputSplit> getSplits(JobContext arg0) throws IOException { FileSystem fs = FileSystem.get(arg0.getConfiguration()); super.setMaxSplitSize(arg0.getConfiguration().getLong("mapred.max.split.size", 471859200)); super.setMinSplitSizeNode(arg0.getConfiguration().getLong("mapred.min.split.size", 471859200)); List<InputSplit> splits = new ArrayList<InputSplit>(); addAllSplits(arg0, Arrays.asList(getInputPaths(arg0)), splits, fs); List<InputSplit> cleanedSplits = new ArrayList<InputSplit>(); for (int i = 0; i < splits.size(); i++) { CombineFileSplit oldSplit = (CombineFileSplit) splits.get(i); String[] locations = oldSplit.getLocations(); if (locations.length > 10) locations = Arrays.copyOf(locations, 10); cleanedSplits.add(new CombineFileSplit(oldSplit.getPaths(), oldSplit.getStartOffsets(), oldSplit.getLengths(), locations)); } return cleanedSplits; }