@Override public Tap getJSONFile( Fields fields, String filename, SinkMode mode ) { return new DirTap( new JSONTextLine( fields ), filename, mode ); }
@Override public Tap getDelimitedFile( Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode ) { return new DirTap( new TextDelimited( fields, hasHeader, delimiter, quote, types ), filename, mode ); }
@Override public Tap getDelimitedFile( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode ) { return new DirTap( new TextDelimited( fields, skipHeader, writeHeader, delimiter, quote, types ), filename, mode ); }
@Override public Tap getDelimitedFile( String delimiter, String quote, FieldTypeResolver fieldTypeResolver, String filename, SinkMode mode ) { return new DirTap( new TextDelimited( true, new DelimitedParser( delimiter, quote, fieldTypeResolver ) ), filename, mode ); }
@Test public void testDirTap() throws Exception { Tap source = new DirTap( new TextLine(), InputData.inputPath, "glob:**/*.txt" ); DirTap sink = new DirTap( new TextLine(), getOutputPath(), SinkMode.REPLACE ); Pipe pipe = new Pipe( "copy" ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); flow.complete(); List<Tuple> list = getSinkAsList( flow ); assertEquals( 674, list.size() ); }
@Override public Fields retrieveSourceFields( FlowProcess<? extends Properties> process, Tap tap ) { if( !skipHeader || !getSourceFields().isUnknown() ) return getSourceFields(); // no need to open them all if( tap instanceof CompositeTap ) tap = (Tap) ( (CompositeTap) tap ).getChildTaps().next(); // can read either a single file, or search the directory for a file if( tap instanceof TapWith ) tap = ( (TapWith) tap ).withScheme( new TextLine( new Fields( "line" ), compressor, charsetName ) ).asTap(); else tap = new DirTap( new TextLine( new Fields( "line" ), compressor, charsetName ), tap.getIdentifier() ); setSourceFields( delimitedParser.parseFirstLine( process, tap ) ); return getSourceFields(); }
@Test public void testSchemeCompression() throws Exception { Tap source = new DirTap( new TextLine(), InputData.inputPath, "glob:**/*.txt" ); DirTap compressed = new DirTap( new TextLine( Compressors.GZIP ), getOutputPath( "compressed" ), SinkMode.REPLACE ); DirTap sink = new DirTap( new TextLine(), getOutputPath( "uncompressed" ), SinkMode.REPLACE ); Flow first = getPlatform().getFlowConnector().connect( "first", source, compressed, new Pipe( "copy" ) ); first.complete(); Flow second = getPlatform().getFlowConnector().connect( "second", compressed, sink, new Pipe( "copy" ) ); second.complete(); List<Tuple> list = getSinkAsList( second ); assertEquals( 674, list.size() ); } }