@Test public void testCombinedHfs() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputFileLower ); Hfs sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputFileUpper ); // create a CombinedHfs instance on these files Tap source = new MultiSourceTap<Hfs, JobConf, RecordReader>( sourceLower, sourceUpper ); FlowProcess<JobConf> process = getPlatform().getFlowProcess(); JobConf conf = process.getConfigCopy(); // set the combine flag conf.setBoolean( HfsProps.COMBINE_INPUT_FILES, true ); conf.set( "cascading.flow.platform", "hadoop" ); // only supported on mr based platforms // test the input format and the split source.sourceConfInit( process, conf ); InputFormat inputFormat = conf.getInputFormat(); assertEquals( Hfs.CombinedInputFormat.class, inputFormat.getClass() ); InputSplit[] splits = inputFormat.getSplits( conf, 1 ); assertEquals( 1, splits.length ); validateLength( source.openForRead( process ), 10 ); }
@Test public void testCombinedHfs() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceLower = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputFileLower ); Hfs sourceUpper = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputFileUpper ); // create a CombinedHfs instance on these files Tap source = new MultiSourceTap<Hfs, JobConf, RecordReader>( sourceLower, sourceUpper ); FlowProcess<JobConf> process = getPlatform().getFlowProcess(); JobConf conf = process.getConfigCopy(); // set the combine flag conf.setBoolean( HfsProps.COMBINE_INPUT_FILES, true ); conf.set( "cascading.flow.platform", "hadoop" ); // only supported on mr based platforms // test the input format and the split source.sourceConfInit( process, conf ); InputFormat inputFormat = conf.getInputFormat(); assertEquals( Hfs.CombinedInputFormat.class, inputFormat.getClass() ); InputSplit[] splits = inputFormat.getSplits( conf, 1 ); assertEquals( 1, splits.length ); validateLength( source.openForRead( process ), 10 ); }