@Test public void testUnGroup() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testRenameNamed() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "item", "element" ), getOutputPath( "rename" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num", "char" ), new Fields( "item", "element" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testUnGroup() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungrouped" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testRenameNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "item" ), new Fields( "char", "item" ), getOutputPath( "renamenarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num" ), new Fields( "item" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\w+\\s\\d+$" ) ); }
@Test public void testRenameNamed() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "item", "element" ), getOutputPath( "rename" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num", "char" ), new Fields( "item", "element" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testRenameNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "item" ), new Fields( "char", "item" ), getOutputPath( "renamenarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, new Fields( "num" ), new Fields( "item" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\w+\\s\\d+$" ) ); }
@Test public void testDiscardNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "num" ), new Fields( "num" ), getOutputPath( "discardnarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Discard( pipe, new Fields( "char" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+$" ) ); }
@Test(expected = FlowTapException.class) public void testTapKeep() throws IOException { getPlatform().copyFromLocal( inputFileCrossX2 ); Tap source = getPlatform().getDelimitedFile( new Fields( "number", "lower", "upper" ), " ", inputFileCrossX2 ); String outputPath = getOutputPath( "/sink" ); Tap sink = getPlatform().getDelimitedFile( new Fields( "upper" ), "+", outputPath, SinkMode.REPLACE ); Flow firstFlow = getPlatform().getFlowConnector().connect( "first", source, sink, new Pipe( "head" ) ); firstFlow.complete(); sink = getPlatform().getDelimitedFile( new Fields( "upper" ), "+", outputPath, SinkMode.KEEP ); Flow secondFlow = getPlatform().getFlowConnector().connect( "second", source, sink, new Each( new Pipe( "head" ), new Debug() ) ); secondFlow.complete(); }
@Test public void testCoerce() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "num", "char" ), getOutputPath( "coerce" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "coerce" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Coerce( pipe, new Fields( "num" ), Integer.class ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testDiscardNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "num" ), new Fields( "num" ), getOutputPath( "discardnarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Discard( pipe, new Fields( "char" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+$" ) ); }
@Test public void testRenameAll() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "item", "element" ), getOutputPath( "renameall" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, Fields.ALL, new Fields( "item", "element" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testUnGroupAnon() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungroupedanon" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testCoerce() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "num", "char" ), getOutputPath( "coerce" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "coerce" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Coerce( pipe, new Fields( "num" ), Integer.class ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testUnique() throws IOException { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "item" ), new Fields( "num", "char" ), getOutputPath( "unique" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Unique( pipe, new Fields( "num" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testRetainNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "num" ), new Fields( "num" ), getOutputPath( "retainnarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Retain( pipe, new Fields( "num" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+$" ) ); }
@Test public void testRenameAll() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "line" ), new Fields( "item", "element" ), getOutputPath( "renameall" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Rename( pipe, Fields.ALL, new Fields( "item", "element" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test public void testUnique() throws IOException { getPlatform().copyFromLocal( inputFileLhs ); Tap source = getPlatform().getTextFile( inputFileLhs ); Tap sink = getPlatform().getTextFile( new Fields( "item" ), new Fields( "num", "char" ), getOutputPath( "unique" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Unique( pipe, new Fields( "num" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+\\s\\w+$" ) ); }
@Test(expected = FlowTapException.class) public void testTapKeep() throws IOException { getPlatform().copyFromLocal( inputFileCrossX2 ); Tap source = getPlatform().getDelimitedFile( new Fields( "number", "lower", "upper" ), " ", inputFileCrossX2 ); String outputPath = getOutputPath( "/sink" ); Tap sink = getPlatform().getDelimitedFile( new Fields( "upper" ), "+", outputPath, SinkMode.REPLACE ); Flow firstFlow = getPlatform().getFlowConnector().connect( "first", source, sink, new Pipe( "head" ) ); firstFlow.complete(); sink = getPlatform().getDelimitedFile( new Fields( "upper" ), "+", outputPath, SinkMode.KEEP ); Flow secondFlow = getPlatform().getFlowConnector().connect( "second", source, sink, new Each( new Pipe( "head" ), new Debug() ) ); secondFlow.complete(); }
@Test public void testUnGroupAnon() throws Exception { getPlatform().copyFromLocal( inputFileJoined ); Tap source = getPlatform().getTextFile( inputFileJoined ); Tap sink = getPlatform().getTextFile( getOutputPath( "ungroupedanon" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) ); pipe = new Each( pipe, new UnGroup( new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 10 ); }
@Test public void testRetainNarrow() throws IOException { getPlatform().copyFromLocal( inputFileLower ); Tap source = getPlatform().getTextFile( inputFileLower ); Tap sink = getPlatform().getTextFile( new Fields( "num" ), new Fields( "num" ), getOutputPath( "retainnarrow" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "shape" ); Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " ); pipe = new Each( pipe, new Fields( "line" ), splitter ); pipe = new Retain( pipe, new Fields( "num" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); validateLength( flow, 5, 1, Pattern.compile( "^\\d+$" ) ); }