/** * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} * @param conf Configuration for the {@link MRInputLegacy} * @param inputFormat InputFormat derived class * @return {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} */ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, Class<?> inputFormat) { return MRInput.createConfigBuilder(conf, inputFormat).setInputClassName(MRInputLegacy.class.getName()); }
/** * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} for a FileInputFormat * @param conf Configuration for the {@link MRInputLegacy} * @param inputFormat FileInputFormat derived class * @param inputPaths Comma separated input paths * @return {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} */ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, Class<?> inputFormat, String inputPaths) { return MRInput.createConfigBuilder(conf, inputFormat, inputPaths).setInputClassName( MRInputLegacy.class.getName()); }
inputConf.set("mapred.input.format.class", TextInputFormat.class.getName()); inputConf.set(FileInputFormat.INPUT_DIR, inputPath); MRInput.MRInputConfigBuilder configurer = MRInput.createConfigBuilder(inputConf, null); DataSourceDescriptor dataSource = configurer.generateSplitsInAM(false).build();
MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath1) .groupSplits(false).build()); Vertex v2 = Vertex.create(VERTEX2, ProcessorDescriptor.create(TokenProcessor.class.getName())); v2.addDataSource(INPUT, MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath2) .groupSplits(false).build()); Vertex v3 = Vertex.create(VERTEX3, ProcessorDescriptor.create(TokenProcessor.class.getName())); v3.addDataSource(INPUT, MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath3) .groupSplits(false).build()); CartesianProductConfig cartesianProductConfig;
ForwardingProcessor.class.getName())).addDataSource("lhs", MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, lhs.toUri().toString()).groupSplits(!isDisableSplitGrouping()) .generateSplitsInAM(!isGenerateSplitInClient()).build()); ForwardingProcessor.class.getName())).addDataSource("rhs", MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, rhs.toUri().toString()).groupSplits(!isDisableSplitGrouping()) .generateSplitsInAM(!isGenerateSplitInClient()).build());
inputFile, MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, hashPath.toUri().toString()) .groupSplits(!isDisableSplitGrouping()) inputFile, MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, streamPath.toUri().toString()) .groupSplits(!isDisableSplitGrouping())
MRInput.MRInputConfigBuilder configBuilder = MRInput.createConfigBuilder( sourceConf, null );
public static DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath, int numPartitions, boolean disableSplitGrouping, boolean isGenerateSplitInClient, String dagName) throws IOException { DataSourceDescriptor dataSource = MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath).groupSplits(!disableSplitGrouping) .generateSplitsInAM(!isGenerateSplitInClient).build();
DataSourceDescriptor dataSource = MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath).groupSplits(!isDisableSplitGrouping()) .generateSplitsInAM(!isGenerateSplitInClient()).build();
MRInput.MRInputConfigBuilder configBuilder = MRInput.createConfigBuilder( sourceConf, null );
inputFile, MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath1.toUri().toString()) .groupSplits(!isDisableSplitGrouping()) inputFile, MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath2.toUri().toString()) .groupSplits(!isDisableSplitGrouping())