@Test public void testDirTap() throws Exception { Tap source = new DirTap( new TextLine(), InputData.inputPath, "glob:**/*.txt" ); DirTap sink = new DirTap( new TextLine(), getOutputPath(), SinkMode.REPLACE ); Pipe pipe = new Pipe( "copy" ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); List<Tuple> list = getSinkAsList( flow ); assertEquals( 674, list.size() ); }
@Test public void testSourceConfInit() throws IOException { getPlatform().copyFromLocal( inputFileNums20 ); Scheme scheme = new SchemeWithProperties( new Fields( "line" ) ); Tap source = getPlatform().getTap( scheme, inputFileNums20, SinkMode.KEEP ); Pipe pipe = new Pipe( "test" ); Tap sink = getPlatform().getTextFile( getOutputPath( "sourceconfinit" ), SinkMode.REPLACE ); Properties properties = new Properties(); properties.setProperty( "default", "connector-default" ); properties.setProperty( "replace", "connector-replace" ); Flow flow = getPlatform().getFlowConnector( properties ).connect( source, sink, pipe ); flow.complete(); assertTrue( flow.resourceExists( sink ) ); }
@Test public void testIO() { String lines = "line1\nline2\n"; System.setIn( new ByteArrayInputStream( lines.getBytes() ) ); ByteArrayOutputStream output = new ByteArrayOutputStream(); System.setOut( new PrintStream( output ) ); Tap source = new StdInTap( new TextLine( new Fields( "line" ) ) ); Tap sink = new StdOutTap( new TextLine( new Fields( "line" ) ) ); Pipe pipe = new Pipe( "io" ); Flow flow = new LocalFlowConnector().connect( source, sink, pipe ); flow.complete(); assertEquals( lines, output.toString() ); }
@Test public void testSchemeCompression() throws Exception { Tap source = new DirTap( new TextLine(), InputData.inputPath, "glob:**/*.txt" ); DirTap compressed = new DirTap( new TextLine( Compressors.GZIP ), getOutputPath( "compressed" ), SinkMode.REPLACE ); DirTap sink = new DirTap( new TextLine(), getOutputPath( "uncompressed" ), SinkMode.REPLACE ); Flow first = getPlatform().getFlowConnector().connect( "first", source, compressed, new Pipe( "copy" ) ); first.complete(); Flow second = getPlatform().getFlowConnector().connect( "second", compressed, sink, new Pipe( "copy" ) ); second.complete(); List<Tuple> list = getSinkAsList( second ); assertEquals( 674, list.size() ); } }
@Test public void testSinkConfInit() throws IOException { getPlatform().copyFromLocal( inputFileNums20 ); Tap source = getPlatform().getTextFile( new Fields( "line" ), inputFileNums20, SinkMode.KEEP ); Pipe pipe = new Pipe( "test" ); Scheme scheme = new SchemeWithProperties( new Fields( "line" ) ); Tap sink = getPlatform().getTap( scheme, getOutputPath( "sinkconfinit" ), SinkMode.REPLACE ); Properties properties = new Properties(); properties.setProperty( "default", "connector-default" ); properties.setProperty( "replace", "connector-replace" ); Flow flow = getPlatform().getFlowConnector( properties ).connect( source, sink, pipe ); flow.complete(); assertTrue( flow.resourceExists( sink ) ); }