@Test public void testFirstNBufferReverse() throws Exception { Set<Tuple> expected = new HashSet<>(); expected.add( new Tuple( "1", "c" ) ); expected.add( new Tuple( "2", "d" ) ); expected.add( new Tuple( "3", "c" ) ); expected.add( new Tuple( "4", "d" ) ); expected.add( new Tuple( "5", "e" ) ); runFirstNBuffer( expected, true, true ); }
private void runTest( Fields sortFields, List<Tuple> result, Comparator defaultComparator ) { Fields fields = new Fields( "a", "b", "c" ); List<Tuple> list = new ArrayList<Tuple>(); list.add( new Tuple( "2", "10", "1" ) ); list.add( new Tuple( "1", "1", "1" ) ); list.add( new Tuple( "3", "1", "1" ) ); Collections.sort( list, new SparseTupleComparator( fields, sortFields, defaultComparator ) ); assertEquals( result, list ); } }
@Test public void testGeneratorAggregator() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new TestAggregator( new Fields( "count1" ), new Fields( "ip" ), new Tuple( "first1" ), new Tuple( "first2" ) ) ); pipe = new Every( pipe, new TestAggregator( new Fields( "count2" ), new Fields( "ip" ), new Tuple( "second" ), new Tuple( "second2" ), new Tuple( "second3" ) ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "generatoraggregator" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8 * 2 * 3, null ); }
@Test public void testLast() { Aggregator aggregator = new Last(); Tuple[] arguments = new Tuple[]{new Tuple( new Double( 1.0 ) ), new Tuple( new Double( 3.0 ) ), new Tuple( new Double( 2.0 ) ), new Tuple( new Double( 4.0 ) ), new Tuple( new Double( -5.0 ) )}; Fields resultFields = new Fields( "field" ); TupleListCollector resultEntryCollector = invokeAggregator( aggregator, arguments, resultFields ); Tuple tuple = resultEntryCollector.iterator().next(); assertEquals( "got expected value after aggregate", -5.0, tuple.getDouble( 0 ), 0.0d ); }
@Test public void testGeneratorAggregator() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new TestAggregator( new Fields( "count1" ), new Fields( "ip" ), new Tuple( "first1" ), new Tuple( "first2" ) ) ); pipe = new Every( pipe, new TestAggregator( new Fields( "count2" ), new Fields( "ip" ), new Tuple( "second" ), new Tuple( "second2" ), new Tuple( "second3" ) ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "generatoraggregator" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8 * 2 * 3, null ); }
@Test public void testMaxValueNonNumber() { Aggregator aggregator = new MaxValue(); Tuple[] arguments = new Tuple[]{new Tuple( 'a' ), new Tuple( 'b' ), new Tuple( 'c' ), new Tuple( 'd' ), new Tuple( 'e' )}; Fields resultFields = new Fields( "field" ); TupleListCollector resultEntryCollector = invokeAggregator( aggregator, arguments, resultFields ); Tuple tuple = resultEntryCollector.iterator().next(); assertEquals( "got expected value after aggregate", 'e', tuple.getChar( 0 ) ); }
@Test public void testCompare4() { Tuple aTuple = new Tuple( "Just My Luck", "ClaudiaPuig", null, "LisaRose", null ); Tuple bTuple = new Tuple( "Just My Luck", "ClaudiaPuig", null, "LisaRose", null ); assertEquals( "not equal: aTuple", bTuple, aTuple ); assertTrue( "not equal than: aTuple = bTuple", aTuple.compareTo( bTuple ) == 0 ); bTuple = new Tuple( "Just My Luck", "ClaudiaPuig", null, "Z", null ); assertTrue( "not less than: aTuple < bTuple", aTuple.compareTo( bTuple ) == "LisaRose".compareTo( "Z" ) ); assertTrue( "not less than: bTuple > aTuple", bTuple.compareTo( aTuple ) == "Z".compareTo( "LisaRose" ) ); }
@Test public void testLastEachNotModified() throws Exception { copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "insert" ), new Tuple( "inserted" ) ) ); pipe = new GroupBy( pipe, new Fields( "insert" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "lasteachmodified" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10, null ); }
@Test public void testMaxValue() { Aggregator aggregator = new MaxValue(); Tuple[] arguments = new Tuple[]{new Tuple( new Double( 1.0 ) ), new Tuple( new Double( 3.0 ) ), new Tuple( new Double( 2.0 ) ), new Tuple( new Double( 4.0 ) ), new Tuple( new Double( -5.0 ) )}; Fields resultFields = new Fields( "field" ); TupleListCollector resultEntryCollector = invokeAggregator( aggregator, arguments, resultFields ); Tuple tuple = resultEntryCollector.iterator().next(); assertEquals( "got expected value after aggregate", 4.0, tuple.getDouble( 0 ), 0.0d ); }
@Test public void testNull() { ValueAssertion assertion = new AssertNull(); assertFail( assertion, getEntry( new Tuple( 1 ) ) ); assertPass( assertion, getEntry( new Tuple( (Comparable) null ) ) ); assertFail( assertion, getEntry( new Tuple( "0", 1 ) ) ); assertFail( assertion, getEntry( new Tuple( "0", null ) ) ); // all values must be null assertPass( assertion, getEntry( new Tuple( null, null ) ) ); }
@Test public void testLastEachNotModified() throws Exception { copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "insert" ), new Tuple( "inserted" ) ) ); pipe = new GroupBy( pipe, new Fields( "insert" ) ); Tap sink = getPlatform().getTextFile( getOutputPath( "lasteachmodified" ), SinkMode.REPLACE ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10, null ); }
@Test public void testCount() { Aggregator aggregator = new Count(); Tuple[] arguments = new Tuple[]{new Tuple( new Double( 1.0 ) ), new Tuple( new Double( 3.0 ) ), new Tuple( new Double( 2.0 ) ), new Tuple( new Double( 4.0 ) ), new Tuple( new Double( -5.0 ) )}; Fields resultFields = new Fields( "field" ); TupleListCollector resultEntryCollector = invokeAggregator( aggregator, arguments, resultFields ); Tuple tuple = resultEntryCollector.iterator().next(); assertEquals( "got expected value after aggregate", 5, tuple.getInteger( 0 ) ); }
@Test public void testExpression() { ValueAssertion assertion = new AssertExpression( "$0 == 1", int.class ); assertPass( assertion, getEntry( new Tuple( 1 ) ) ); assertFail( assertion, getEntry( new Tuple( (Comparable) null ) ) ); assertPass( assertion, getEntry( new Tuple( "1", 0 ) ) ); assertFail( assertion, getEntry( new Tuple( "0", null ) ) ); } }
@Test public void testSimpleBuffer3() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simple3" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ), "\\s" ) ); pipe = new GroupBy( pipe, new Fields( "num" ) ); pipe = new Every( pipe, new TestBuffer( new Fields( "new" ), new Tuple( "new" ) ), new Fields( "new", "lower", "upper" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, null ); List<Tuple> results = getSinkAsList( flow ); assertTrue( results.contains( new Tuple( "new\ta\tA" ) ) ); assertTrue( results.contains( new Tuple( "new\tb\tB" ) ) ); assertTrue( results.contains( new Tuple( "new\tc\tC" ) ) ); }
@Test public void testSum() { Aggregator aggregator = new Sum(); Tuple[] arguments = new Tuple[]{new Tuple( new Double( 1.0 ) ), new Tuple( new Double( 3.0 ) ), new Tuple( new Double( 2.0 ) ), new Tuple( new Double( 4.0 ) ), new Tuple( new Double( -5.0 ) )}; Fields resultFields = new Fields( "field" ); TupleListCollector resultEntryCollector = invokeAggregator( aggregator, arguments, resultFields ); Tuple tuple = resultEntryCollector.iterator().next(); assertEquals( "got expected value after aggregate", 5.0, tuple.getDouble( 0 ), 0.0d ); }
@Test public void testTupleMoreThan() { ValueAssertion assertion = new AssertSizeMoreThan( 1 ); assertFail( assertion, getEntry( new Tuple( 1 ) ) ); assertFail( assertion, getEntry( new Tuple( (Comparable) null ) ) ); assertPass( assertion, getEntry( new Tuple( "0", 1 ) ) ); assertPass( assertion, getEntry( new Tuple( "0", null ) ) ); } }