protected void applySourceConfInitIdentifiers( FlowProcess<? extends Configuration> process, Configuration conf, final String... fullIdentifiers ) { sourceConfInitAddInputPaths( conf, new LazyIterable<String, Path>( fullIdentifiers ) { @Override protected Path convert( String next ) { return new Path( next ); } } ); sourceConfInitComplete( process, conf ); }
protected void applySourceConfInitIdentifiers( FlowProcess<? extends Configuration> process, Configuration conf, final String... fullIdentifiers ) { sourceConfInitAddInputPaths( conf, new LazyIterable<String, Path>( fullIdentifiers ) { @Override protected Path convert( String next ) { return new Path( next ); } } ); sourceConfInitComplete( process, conf ); }
@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { Hfs[] taps = initTapsInternal( conf ); taps[ 0 ].sourceConfInitAddInputPaths( conf, new LazyIterable<Hfs, Path>( taps ) { @Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } ); taps[ 0 ].sourceConfInitComplete( process, conf ); }
@Override public void sourceConfInit( FlowProcess<? extends Configuration> process, Configuration conf ) { Hfs[] taps = initTapsInternal( conf ); taps[ 0 ].sourceConfInitAddInputPaths( conf, new LazyIterable<Hfs, Path>( taps ) { @Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } ); taps[ 0 ].sourceConfInitComplete( process, conf ); }
private void registerHfs( FlowProcess<? extends Configuration> process, Configuration conf, Hfs hfs ) throws IOException { if( isSimpleGlob() ) { FileSystem fs = FileSystem.get( conf ); FileStatus[] statuses = fs.globStatus( getHfs().getPath() ); if( statuses == null || statuses.length == 0 ) throw new TapException( String.format( "glob expression %s does not match any files on the filesystem", getHfs().getPath() ) ); for( FileStatus fileStatus : statuses ) registerURI( conf, fileStatus.getPath() ); } else { registerURI( conf, hfs.getPath() ); } hfs.sourceConfInitComplete( process, conf ); }
private void registerHfs( FlowProcess<? extends Configuration> process, Configuration conf, Hfs hfs ) throws IOException { if( isSimpleGlob() ) { FileSystem fs = FileSystem.get( conf ); FileStatus[] statuses = fs.globStatus( getHfs().getPath() ); if( statuses == null || statuses.length == 0 ) throw new TapException( String.format( "glob expression %s does not match any files on the filesystem", getHfs().getPath() ) ); for( FileStatus fileStatus : statuses ) registerURI( conf, fileStatus.getPath() ); } else { registerURI( conf, hfs.getPath() ); } hfs.sourceConfInitComplete( process, conf ); }