@Test public void testLfs() throws URISyntaxException, IOException { Tap tap = new Lfs( new SequenceFile( new Fields( "foo" ) ), "some/path" ); String path = tap.getFullIdentifier( getPlatform().getFlowProcess() ); assertTrue( "wrong scheme", new Path( path ).toUri().getScheme().equalsIgnoreCase( "file" ) ); new Lfs( new SequenceFile( new Fields( "foo" ) ), "file:///some/path" ); try { new Lfs( new SequenceFile( new Fields( "foo" ) ), "s3://localhost:5001/some/path" ); fail( "not valid url" ); } catch( Exception exception ) { } }
@Test public void testLfs() throws URISyntaxException, IOException { Tap tap = new Lfs( new SequenceFile( new Fields( "foo" ) ), "some/path" ); String path = tap.getFullIdentifier( getPlatform().getFlowProcess() ); assertTrue( "wrong scheme", new Path( path ).toUri().getScheme().equalsIgnoreCase( "file" ) ); new Lfs( new SequenceFile( new Fields( "foo" ) ), "file:///some/path" ); try { new Lfs( new SequenceFile( new Fields( "foo" ) ), "s3://localhost:5001/some/path" ); fail( "not valid url" ); } catch( Exception exception ) { } }
@Test public void testHfsAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "*" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
@Test public void testHfsBracketAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "{*}" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
@Test public void testHfsBracketAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "{*}" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
@Test public void testHfsAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "*" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
@Test public void testMissingInputFormat() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = new Hfs( new TextDelimited( new Fields( "offset", "line" ) ), inputFileApache ) { @Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { // don't set input format //super.sourceConfInit( process, conf ); } }; Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = new Hfs( new TextDelimited( Fields.ALL ), getOutputPath( "missinginputformat" ), SinkMode.REPLACE ); try { Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); fail( "did not test for missing input format" ); } catch( Exception exception ) { // ignore } }
@Test public void testMissingInputFormat() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = new Hfs( new TextDelimited( new Fields( "offset", "line" ) ), inputFileApache ) { @Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { // don't set input format //super.sourceConfInit( process, conf ); } }; Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = new Hfs( new TextDelimited( Fields.ALL ), getOutputPath( "missinginputformat" ), SinkMode.REPLACE ); try { Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); fail( "did not test for missing input format" ); } catch( Exception exception ) { // ignore } }