@Test public void testBuildMerge() { Tap sourceLower = getPlatform().getTextFile( "file1" ); Tap sourceUpper = getPlatform().getTextFile( "file2" ); Map sources = new HashMap(); sources.put( "lower", sourceLower ); sources.put( "upper", sourceUpper ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); Tap sink = getPlatform().getTextFile( "outpath", SinkMode.REPLACE ); Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter ); Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter ); Pipe splice = new GroupBy( "merge", Pipe.pipes( pipeLower, pipeUpper ), new Fields( "num" ), null, false ); Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice ); }
@Test public void testBuildMerge() { Tap sourceLower = getPlatform().getTextFile( "file1" ); Tap sourceUpper = getPlatform().getTextFile( "file2" ); Map sources = new HashMap(); sources.put( "lower", sourceLower ); sources.put( "upper", sourceUpper ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); Tap sink = getPlatform().getTextFile( "outpath", SinkMode.REPLACE ); Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter ); Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter ); Pipe splice = new GroupBy( "merge", Pipe.pipes( pipeLower, pipeUpper ), new Fields( "num" ), null, false ); Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice ); }
@Test public void testUnGroup() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testSwap() throws Exception { Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Tap sink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "count", "ipaddress" ), getOutputPath( "swap" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); Function parser = new RegexParser( new Fields( "ip" ), "^[^ ]*" ); pipe = new Each( pipe, new Fields( "line" ), parser, Fields.SWAP ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Fields( "ip" ), new Count( new Fields( "count" ) ) ); pipe = new Each( pipe, new Fields( "ip" ), new Identity( new Fields( "ipaddress" ) ), Fields.SWAP ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8, 2, Pattern.compile( "^\\d+\\s\\d+\\s[\\d]{1,3}\\.[\\d]{1,3}\\.[\\d]{1,3}\\.[\\d]{1,3}$" ) ); }
@Test public void testUnGroup() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testRenameNamed() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "item", "element" ), getOutputPath( "rename" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num", "char" ), new Fields( "item", "element" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testRenameNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "item" ), new Fields( "char", "item" ), getOutputPath( "renamenarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num" ), new Fields( "item" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\w+\\s\\d+$" ) ); }
@Test public void testDiscardNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "num" ), new Fields( "num" ), getOutputPath( "discardnarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Discard( pipe, new Fields( "char" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+$" ) ); }
private Flow firstFlow( String name ) { Tap source = getPlatform().getTextFile( inputFileIps ); Pipe pipe = new Pipe( name ); pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) ); Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( name ), SinkMode.REPLACE ); return getPlatform().getFlowConnector().connect( source, sink, pipe ); }
private Flow firstFlow( String path ) { Tap source = getPlatform().getTextFile( inputFileIps ); Pipe pipe = new Pipe( "first" ); pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) ); Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( path + "/first" ), SinkMode.REPLACE ); return getPlatform().getFlowConnector().connect( source, sink, pipe ); }
@Test public void testRenameNamed() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "item", "element" ), getOutputPath( "rename" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num", "char" ), new Fields( "item", "element" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testSwap() throws Exception { Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache ); Tap sink = getPlatform().getTextFile( new Fields( "offset", "line" ), new Fields( "count", "ipaddress" ), getOutputPath( "swap" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); Function parser = new RegexParser( new Fields( "ip" ), "^[^ ]*" ); pipe = new Each( pipe, new Fields( "line" ), parser, Fields.SWAP ); pipe = new GroupBy( pipe, new Fields( "ip" ) ); pipe = new Every( pipe, new Fields( "ip" ), new Count( new Fields( "count" ) ) ); pipe = new Each( pipe, new Fields( "ip" ), new Identity( new Fields( "ipaddress" ) ), Fields.SWAP ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 8, 2, Pattern.compile( "^\\d+\\s\\d+\\s[\\d]{1,3}\\.[\\d]{1,3}\\.[\\d]{1,3}\\.[\\d]{1,3}$" ) ); }
private Flow firstFlow( String path ) { Tap source = getPlatform().getTextFile( inputFileIps ); Pipe pipe = new Pipe( "first" ); pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) ); Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( path + "/first" ), SinkMode.REPLACE ); return getPlatform().getFlowConnector().connect( source, sink, pipe ); }
@Test public void testRenameNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "item" ), new Fields( "char", "item" ), getOutputPath( "renamenarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num" ), new Fields( "item" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\w+\\s\\d+$" ) ); }
@Test public void testUnGroupAnon() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungroupedanon" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
private Flow previousMultiTapFlow( String path, String ordinal ) { Tap source = getPlatform().getTextFile( inputFileIps ); Pipe pipe = new Pipe( ordinal ); pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) ); Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( path + "/" + ordinal ), SinkMode.REPLACE ); return getPlatform().getFlowConnector().connect( "previous-multi-tap-" + ordinal, source, sink, pipe ); }
private Flow previousMultiTapFlow( String path, String ordinal ) { Tap source = getPlatform().getTextFile( inputFileIps ); Pipe pipe = new Pipe( ordinal ); pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) ); Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( path + "/" + ordinal ), SinkMode.REPLACE ); return getPlatform().getFlowConnector().connect( "previous-multi-tap-" + ordinal, source, sink, pipe ); }
@Test public void testUnGroupAnon() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungroupedanon" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testDiscardNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "num" ), new Fields( "num" ), getOutputPath( "discardnarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Discard( pipe, new Fields( "char" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+$" ) ); }
@Test public void testRenameAll() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "item", "element" ), getOutputPath( "renameall" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, Fields.ALL, new Fields( "item", "element" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }