TupleEntryIterator iterator = flow.openSource();
TupleEntryIterator iterator = flow.openSource();
@Test public void testNullsFromScheme() throws IOException { getPlatform().copyFromLocal( inputFileComments ); Tap source = new Hfs( new CommentScheme( new Fields( "line" ) ), inputFileComments ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Identity() ); Tap sink = new Hfs( new TextLine( 1 ), getOutputPath( "testnulls" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector( getProperties() ).connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, null ); TupleEntryIterator iterator = flow.openSink(); assertEquals( "not equal: tuple.get(1)", "1 a", iterator.next().getObject( 1 ) ); iterator.close(); // confirm the tuple iterator can handle nulls from the source validateLength( flow.openSource(), 5 ); }
@Test public void testNullsFromScheme() throws IOException { getPlatform().copyFromLocal( inputFileComments ); Tap source = new Hfs( new CommentScheme( new Fields( "line" ) ), inputFileComments ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Identity() ); Tap sink = new Hfs( new TextLine( 1 ), getOutputPath( "testnulls" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector( getProperties() ).connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, null ); TupleEntryIterator iterator = flow.openSink(); assertEquals( "not equal: tuple.get(1)", "1 a", iterator.next().getObject( 1 ) ); iterator.close(); // confirm the tuple iterator can handle nulls from the source validateLength( flow.openSource(), 5 ); }
@Test public void testSimpleGroup() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "simple" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 8, null ); }
@Test public void testSimpleGroup() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "simple" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 8, null ); }
@Test public void testSkipStrategiesKeep() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "keep" ), SinkMode.KEEP ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", new FlowSkipIfSinkExists().skipFlow( flow ) ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }
@Test public void testSkipStrategiesKeep() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "keep" ), SinkMode.KEEP ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", new FlowSkipIfSinkExists().skipFlow( flow ) ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }
validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 8, null );
validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 8, null );
@Test public void testSkipStrategiesReplace() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "replace" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); FlowSkipStrategy old = flow.getFlowSkipStrategy(); FlowSkipStrategy replaced = flow.setFlowSkipStrategy( new FlowSkipIfSinkExists() ); assertTrue( "not same instance", old == replaced ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }
@Test public void testSkipStrategiesReplace() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "replace" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); FlowSkipStrategy old = flow.getFlowSkipStrategy(); FlowSkipStrategy replaced = flow.setFlowSkipStrategy( new FlowSkipIfSinkExists() ); assertTrue( "not same instance", old == replaced ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }