public static void addInputPath(Job job, Path path, FormatBundle inputBundle, int nodeIndex) { Configuration conf = job.getConfiguration(); String inputs = JOINER.join(inputBundle.serialize(), String.valueOf(nodeIndex), path.toString()); String existing = conf.get(CRUNCH_INPUTS); conf.set(CRUNCH_INPUTS, existing == null ? inputs : existing + RECORD_SEP + inputs); }
public static void addNamedOutput(Job job, String name, FormatBundle<? extends OutputFormat> outputBundle, Class keyClass, Class valueClass) { Configuration conf = job.getConfiguration(); String inputs = JOINER.join(name, outputBundle.serialize(), keyClass.getName(), valueClass.getName()); String existing = conf.get(CRUNCH_OUTPUTS); conf.set(CRUNCH_OUTPUTS, existing == null ? inputs : existing + RECORD_SEP + inputs); }
public static void addNamedOutput(Job job, String name, FormatBundle<? extends OutputFormat> outputBundle, Class keyClass, Class valueClass) { Configuration conf = job.getConfiguration(); String inputs = JOINER.join(name, outputBundle.serialize(), keyClass.getName(), valueClass.getName()); String existing = conf.get(CRUNCH_OUTPUTS); conf.set(CRUNCH_OUTPUTS, existing == null ? inputs : existing + RECORD_SEP + inputs); }
public static void addInputPaths(Job job, Collection<Path> paths, FormatBundle inputBundle, int nodeIndex) { Configuration conf = job.getConfiguration(); List<String> encodedPathStrs = Lists.newArrayListWithExpectedSize(paths.size()); for (Path path : paths) { String pathStr = encodePath(path); Preconditions.checkArgument(pathStr.indexOf(RECORD_SEP) == -1 && pathStr.indexOf(FIELD_SEP) == -1 && pathStr.indexOf(PATH_SEP) == -1); encodedPathStrs.add(pathStr); } String inputs = JOINER.join(inputBundle.serialize(), String.valueOf(nodeIndex), Joiner.on(PATH_SEP).join(encodedPathStrs)); String existing = conf.get(CRUNCH_INPUTS); conf.set(CRUNCH_INPUTS, existing == null ? inputs : existing + RECORD_SEP + inputs); }