@Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } );
@Override protected Path convert( Hfs next ) { return next.getPath(); // we are building fully qualified paths above } } );
@Override public String getIdentifier() { if( cachedPath == null ) cachedPath = getPath().toString(); return cachedPath; }
@Override public boolean resourceExists( Configuration conf ) throws IOException { // unfortunately getFileSystem( conf ).exists( getPath() ); does not account for "/*" etc // nor is there an more efficient means to test for existence FileStatus[] fileStatuses = getFileSystem( conf ).globStatus( getPath() ); return fileStatuses != null && fileStatuses.length > 0; }
/** * Given a file-system object, it makes an array of paths * * @param conf of type JobConf * @throws IOException on failure */ private void makeStatuses( Configuration conf ) throws IOException { if( statuses != null ) return; statuses = getFileSystem( conf ).listStatus( getPath() ); }
@Override public String getFullIdentifier( Configuration conf ) { return getPath().makeQualified( getFileSystem( conf ) ).toString(); }
/** * Given a file-system object, it makes an array of paths * * @param conf of type JobConf * @throws IOException on failure */ private void makeStatuses( Configuration conf ) throws IOException { if( statuses != null ) return; statuses = getFileSystem( conf ).listStatus( getPath() ); }
@Override public String getFullIdentifier( Configuration conf ) { return getPath().makeQualified( getFileSystem( conf ) ).toString(); }
@Override public boolean createResource( Configuration conf ) throws IOException { if( LOG.isDebugEnabled() ) LOG.debug( "making dirs: {}", getFullIdentifier( conf ) ); return getFileSystem( conf ).mkdirs( getPath() ); }
@Override public boolean createResource( Configuration conf ) throws IOException { if( LOG.isDebugEnabled() ) LOG.debug( "making dirs: {}", getFullIdentifier( conf ) ); return getFileSystem( conf ).mkdirs( getPath() ); }
@Override public boolean commitResource( Configuration conf ) throws IOException { Hadoop18TapUtil.writeSuccessMarker( conf, ( (Hfs) parent ).getPath() ); return super.commitResource( conf ); } }
@Override public TapWith<Configuration, RecordReader, OutputCollector> withSinkMode( SinkMode sinkMode ) { return create( getScheme(), getPath(), sinkMode ); }
@Override public TapWith<Configuration, RecordReader, OutputCollector> withScheme( Scheme<Configuration, RecordReader, OutputCollector, ?, ?> scheme ) { return create( scheme, getPath(), getSinkMode() ); }
@Override public TapWith<Configuration, RecordReader, OutputCollector> withSinkMode( SinkMode sinkMode ) { return create( getScheme(), getPath(), sinkMode ); }
@Override public boolean isDirectory( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return false; return getFileSystem( conf ).getFileStatus( getPath() ).isDir(); }
@Override public long getSize( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return getFileSystem( conf ).getFileStatus( getPath() ).getLen(); }
@Override public long getSize( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return getFileSystem( conf ).getFileStatus( getPath() ).getLen(); }
private List<Footer> getFooters(FlowProcess<? extends JobConf> flowProcess, Hfs hfs) throws IOException { JobConf jobConf = flowProcess.getConfigCopy(); DeprecatedParquetInputFormat format = new DeprecatedParquetInputFormat(); format.addInputPath(jobConf, hfs.getPath()); return format.getFooters(jobConf); }
private List<Footer> getFooters(FlowProcess<JobConf> flowProcess, Hfs hfs) throws IOException { JobConf jobConf = flowProcess.getConfigCopy(); DeprecatedParquetInputFormat format = new DeprecatedParquetInputFormat(); format.addInputPath(jobConf, hfs.getPath()); return format.getFooters(jobConf); }