getPlatform().setNumGatherPartitionTasks( properties, 1 );
getPlatform().setNumGatherPartitionTasks( properties, 1 );
@Test public void testTrapTapSourceSink() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Scheme scheme = getPlatform().getTestFailScheme(); Tap source = getPlatform().getTap( scheme, inputFileApache, SinkMode.KEEP ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTap( scheme, getOutputPath( "trapsourcesink/sink" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "trapsourcesink/trap" ), SinkMode.REPLACE ); Map<Object, Object> properties = getProperties(); // compensate for running in cluster mode getPlatform().setNumMapTasks( properties, 1 ); getPlatform().setNumReduceTasks( properties, 1 ); getPlatform().setNumGatherPartitionTasks( properties, 1 ); Flow flow = getPlatform().getFlowConnector( properties ).connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow.openTapForRead( getPlatform().getTextFile( sink.getIdentifier() ) ), 7 ); validateLength( flow.openTrap(), 2, Pattern.compile( "bad data" ) ); // confirm the payload is written }
@Test public void testTrapTapSourceSink() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Scheme scheme = getPlatform().getTestFailScheme(); Tap source = getPlatform().getTap( scheme, inputFileApache, SinkMode.KEEP ); Pipe pipe = new Pipe( "map" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); Tap sink = getPlatform().getTap( scheme, getOutputPath( "trapsourcesink/sink" ), SinkMode.REPLACE ); Tap trap = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "trapsourcesink/trap" ), SinkMode.REPLACE ); Map<Object, Object> properties = getProperties(); // compensate for running in cluster mode getPlatform().setNumMapTasks( properties, 1 ); getPlatform().setNumReduceTasks( properties, 1 ); getPlatform().setNumGatherPartitionTasks( properties, 1 ); Flow flow = getPlatform().getFlowConnector( properties ).connect( "trap test", source, sink, trap, pipe ); flow.complete(); validateLength( flow.openTapForRead( getPlatform().getTextFile( sink.getIdentifier() ) ), 7 ); validateLength( flow.openTrap(), 2, Pattern.compile( "bad data" ) ); // confirm the payload is written }