Refine search
@Test public void testSimpleChain() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count( new Fields( "count1" ) ) ); pipe = new Every( pipe, new Count( new Fields( "count2" ) ) ); pipe = new Every( pipe, new Count( new Fields( "count3" ) ) ); pipe = new Every( pipe, new Count( new Fields( "count4" ) ) ); Tap sink = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "simplechain" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8, 5 ); }
@Override public void complete( FlowProcess flowProcess, AggregatorCall<Pair<Long[], Tuple>> aggregatorCall ) { aggregatorCall.getOutputCollector().add( getResult( aggregatorCall ) ); }
void runTestCount( String name, Fields argumentSelector, Fields fieldDeclaration, Fields outputSelector ) throws Exception { getPlatform().copyFromLocal( inputFileIps ); Tap source = getPlatform().getTextFile( Fields.size( 2 ), inputFileIps ); Tap sink = getPlatform().getTextFile( Fields.size( 1 ), getOutputPath( name ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, argumentSelector, new Count( fieldDeclaration ), outputSelector ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.start(); // simple test for start flow.complete(); validateLength( flow, 17 ); assertTrue( getSinkAsList( flow ).contains( new Tuple( "63.123.238.8\t2" ) ) ); }
@Test public void testName() { Pipe count = new Pipe( "count" ); Pipe pipe = new GroupBy( count, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) ); assertEquals( "not equal: count.getName()", "count", count.getName() ); assertEquals( "not equal: pipe.getName()", "count", pipe.getName() ); pipe = new Each( count, new Fields( 1 ), new RegexSplitter( Fields.size( 2 ) ) ); assertEquals( "not equal: pipe.getName()", "count", pipe.getName() ); }
@Test public void testSimpleGroup() throws Exception { getPlatform().copyFromLocal( inputFileApache200 ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache200 ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); pipe = new Each( pipe, new Fields( "ip" ), new Stop( new Limit( 100 ) ) ); Tap sink = getPlatform().getTextFile( getOutputPath(), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow.openSink(), 100 ); assertEquals( 100, flow.getFlowStats().getCounterValue( StepCounters.Tuples_Written ) ); assertEquals( 200, flow.getFlowStats().getCounterValue( StepCounters.Tuples_Read ) ); }
@Test public void testEveryOutResolver() throws Exception { Fields sourceFields = new Fields( "first", "second" ); Tap source = getPlatform().getTabDelimitedFile( sourceFields, "input/path", SinkMode.KEEP ); Fields sinkFields = new Fields( "third", "fourth" ); Tap sink = getPlatform().getTabDelimitedFile( sinkFields, "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new GroupBy( pipe, new Fields( "first" ) ); pipe = new Every( pipe, new Fields( "second" ), new Count(), new Fields( "third" ) ); verify( source, sink, pipe ); }
@Override public List<Pipe> resolveTails( Context context ) { Pipe pipe = new Pipe( (String) context.getFlow().getSourceNames().get( 0 ) ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); return Arrays.asList( pipe ); } };
var Count = function() { this.one = 1; this.two = 2; this.three = 3; } var obj = new Count(); obj.four = 4;
void runTestCount( String name, Fields argumentSelector, Fields fieldDeclaration, Fields outputSelector ) throws Exception { getPlatform().copyFromLocal( inputFileIps ); Tap source = getPlatform().getTextFile( Fields.size( 2 ), inputFileIps ); Tap sink = getPlatform().getTextFile( Fields.size( 1 ), getOutputPath( name ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "count" ); pipe = new GroupBy( pipe, new Fields( 1 ) ); pipe = new Every( pipe, argumentSelector, new Count( fieldDeclaration ), outputSelector ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.start(); // simple test for start flow.complete(); validateLength( flow, 17 ); assertTrue( getSinkAsList( flow ).contains( new Tuple( "63.123.238.8\t2" ) ) ); }
@Test public void testName() { Pipe count = new Pipe( "count" ); Pipe pipe = new GroupBy( count, new Fields( 1 ) ); pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) ); assertEquals( "not equal: count.getName()", "count", count.getName() ); assertEquals( "not equal: pipe.getName()", "count", pipe.getName() ); pipe = new Each( count, new Fields( 1 ), new RegexSplitter( Fields.size( 2 ) ) ); assertEquals( "not equal: pipe.getName()", "count", pipe.getName() ); }
@Test public void testEveryOutResolver() throws Exception { Fields sourceFields = new Fields( "first", "second" ); Tap source = getPlatform().getTabDelimitedFile( sourceFields, "input/path", SinkMode.KEEP ); Fields sinkFields = new Fields( "third", "fourth" ); Tap sink = getPlatform().getTabDelimitedFile( sinkFields, "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new GroupBy( pipe, new Fields( "first" ) ); pipe = new Every( pipe, new Fields( "second" ), new Count(), new Fields( "third" ) ); verify( source, sink, pipe ); }
@Override public List<Pipe> resolveTails( Context context ) { Pipe pipe = new Pipe( (String) context.getFlow().getSourceNames().get( 0 ) ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) ); return Arrays.asList( pipe ); } };
function Count() { var c = 1; return function() { return c++; } }; var count = new Count(); // count is now a function that adds and returns count(); // 1 count(); // 2
@Test public void testSimpleChain() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Count( new Fields( "count1" ) ) ); pipe = new Every( pipe, new Count( new Fields( "count2" ) ) ); pipe = new Every( pipe, new Count( new Fields( "count3" ) ) ); pipe = new Every( pipe, new Count( new Fields( "count4" ) ) ); Tap sink = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( "simplechain" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8, 5 ); }
@Test public void testEveryArgResolver() throws Exception { Fields sourceFields = new Fields( "first", "second" ); Tap source = getPlatform().getTabDelimitedFile( sourceFields, "input/path", SinkMode.KEEP ); Fields sinkFields = new Fields( "third", "fourth" ); Tap sink = getPlatform().getTabDelimitedFile( sinkFields, "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new GroupBy( pipe, new Fields( "first" ) ); pipe = new Every( pipe, new Fields( "third" ), new Count() ); verify( source, sink, pipe ); }
@Test public void testCount() { Aggregator aggregator = new Count(); Tuple[] arguments = new Tuple[]{new Tuple( new Double( 1.0 ) ), new Tuple( new Double( 3.0 ) ), new Tuple( new Double( 2.0 ) ), new Tuple( new Double( 4.0 ) ), new Tuple( new Double( -5.0 ) )}; Fields resultFields = new Fields( "field" ); TupleListCollector resultEntryCollector = invokeAggregator( aggregator, arguments, resultFields ); Tuple tuple = resultEntryCollector.iterator().next(); assertEquals( "got expected value after aggregate", 5, tuple.getInteger( 0 ) ); }
@Test public void testSwap() throws Exception { Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Tap sink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "count", "ipaddress" ), getOutputPath( "swap" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); Function parser = new RegexParser( new Fields( "ip" ), "^[^ ]*" ); pipe = new Each( pipe, new Fields( "line" ), parser, Fields.SWAP ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Fields( "ip" ), new Count( new Fields( "count" ) ) ); pipe = new Each( pipe, new Fields( "ip" ), new Identity( new Fields( "ipaddress" ) ), Fields.SWAP ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8, 2, Pattern.compile( "^\\d+\\s\\d+\\s[\\d]{1,3}\\.[\\d]{1,3}\\.[\\d]{1,3}\\.[\\d]{1,3}$" ) ); }
@Test public void testEveryArgResolver() throws Exception { Fields sourceFields = new Fields( "first", "second" ); Tap source = getPlatform().getTabDelimitedFile( sourceFields, "input/path", SinkMode.KEEP ); Fields sinkFields = new Fields( "third", "fourth" ); Tap sink = getPlatform().getTabDelimitedFile( sinkFields, "output/path", SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new GroupBy( pipe, new Fields( "first" ) ); pipe = new Every( pipe, new Fields( "third" ), new Count() ); verify( source, sink, pipe ); }