@Override public boolean source( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) throws IOException { boolean success = sourceCall.getInput().next( sourceCall.getContext()[ 0 ], sourceCall.getContext()[ 1 ] ); if( !success ) return false; if( sourceCall.getContext()[ 1 ].toString().matches( "^\\s*#.*$" ) ) return source( flowProcess, sourceCall ); sourceHandleInput( sourceCall ); return true; } }
@Test public void testNullsFromScheme() throws IOException { getPlatform().copyFromLocal( inputFileComments ); Tap source = new Hfs( new CommentScheme( new Fields( "line" ) ), inputFileComments ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Identity() ); Tap sink = new Hfs( new TextLine( 1 ), getOutputPath( "testnulls" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector( getProperties() ).connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, null ); TupleEntryIterator iterator = flow.openSink(); assertEquals( "not equal: tuple.get(1)", "1 a", iterator.next().getObject( 1 ) ); iterator.close(); // confirm the tuple iterator can handle nulls from the source validateLength( flow.openSource(), 5 ); }
@Test public void testNullsFromScheme() throws IOException { getPlatform().copyFromLocal( inputFileComments ); Tap source = new Hfs( new CommentScheme( new Fields( "line" ) ), inputFileComments ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Identity() ); Tap sink = new Hfs( new TextLine( 1 ), getOutputPath( "testnulls" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector( getProperties() ).connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, null ); TupleEntryIterator iterator = flow.openSink(); assertEquals( "not equal: tuple.get(1)", "1 a", iterator.next().getObject( 1 ) ); iterator.close(); // confirm the tuple iterator can handle nulls from the source validateLength( flow.openSource(), 5 ); }
@Override public boolean source( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) throws IOException { boolean success = sourceCall.getInput().next( sourceCall.getContext()[ 0 ], sourceCall.getContext()[ 1 ] ); if( !success ) return false; if( sourceCall.getContext()[ 1 ].toString().matches( "^\\s*#.*$" ) ) return source( flowProcess, sourceCall ); sourceHandleInput( sourceCall ); return true; } }