@Override public String[] getChildIdentifiers( Configuration conf, int depth, boolean fullyQualified ) throws IOException { if( !resourceExists( conf ) ) return new String[ 0 ]; if( depth == 0 && !fullyQualified ) return new String[]{getIdentifier()}; String fullIdentifier = getFullIdentifier( conf ); int trim = fullyQualified ? 0 : fullIdentifier.length() + 1; Set<String> results = new LinkedHashSet<String>(); getChildPaths( conf, results, trim, new Path( fullIdentifier ), depth ); return results.toArray( new String[ results.size() ] ); }
@Override public String[] getChildIdentifiers( Configuration conf, int depth, boolean fullyQualified ) throws IOException { if( !resourceExists( conf ) ) return new String[ 0 ]; if( depth == 0 && !fullyQualified ) return new String[]{getIdentifier()}; String fullIdentifier = getFullIdentifier( conf ); int trim = fullyQualified ? 0 : fullIdentifier.length() + 1; Set<String> results = new LinkedHashSet<String>(); getChildPaths( conf, results, trim, new Path( fullIdentifier ), depth ); return results.toArray( new String[ results.size() ] ); }
@Override public boolean isDirectory( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return false; return getFileSystem( conf ).getFileStatus( getPath() ).isDir(); }
@Override public boolean isDirectory( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return false; return getFileSystem( conf ).getFileStatus( getPath() ).isDir(); }
@Override public long getSize( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return getFileSystem( conf ).getFileStatus( getPath() ).getLen(); }
@Override public long getSize( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return getFileSystem( conf ).getFileStatus( getPath() ).getLen(); }
@Override public long getModifiedTime( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( !fileStatus.isDir() ) return fileStatus.getModificationTime(); // todo: this should ignore the _temporary path, or not cache if found in the array makeStatuses( conf ); // statuses is empty, return 0 if( statuses == null || statuses.length == 0 ) return 0; long date = 0; // filter out directories as we don't recurs into sub dirs for( FileStatus status : statuses ) { if( !status.isDir() ) date = Math.max( date, status.getModificationTime() ); } return date; }
@Override public long getModifiedTime( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( !fileStatus.isDir() ) return fileStatus.getModificationTime(); // todo: this should ignore the _temporary path, or not cache if found in the array makeStatuses( conf ); // statuses is empty, return 0 if( statuses == null || statuses.length == 0 ) return 0; long date = 0; // filter out directories as we don't recurs into sub dirs for( FileStatus status : statuses ) { if( !status.isDir() ) date = Math.max( date, status.getModificationTime() ); } return date; }
/** * Method getBlockSize returns the {@code blocksize} specified by the underlying file system for this resource. * * @param conf of JobConf * @return long * @throws IOException when */ public long getBlockSize( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return fileStatus.getBlockSize(); }
/** * Method getReplication returns the {@code replication} specified by the underlying file system for * this resource. * * @param conf of JobConf * @return int * @throws IOException when */ public int getReplication( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return fileStatus.getReplication(); }
/** * Method getReplication returns the {@code replication} specified by the underlying file system for * this resource. * * @param conf of JobConf * @return int * @throws IOException when */ public int getReplication( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return fileStatus.getReplication(); }
/** * Method getBlockSize returns the {@code blocksize} specified by the underlying file system for this resource. * * @param conf of JobConf * @return long * @throws IOException when */ public long getBlockSize( Configuration conf ) throws IOException { if( !resourceExists( conf ) ) return 0; FileStatus fileStatus = getFileStatus( conf ); if( fileStatus.isDir() ) return 0; return fileStatus.getBlockSize(); }
public boolean testExists( Flow flow, Hfs tap ) { try { // don't test for _SUCCESS if the tap is a file, only if a directory if( !tap.isDirectory( flow.getFlowProcess() ) ) return true; return new Hfs( new TextLine(), new Path( tap.getPath(), "_SUCCESS" ).toString() ).resourceExists( flow.getFlowProcess() ); } catch( IOException exception ) { throw new FlowException( exception ); } }
public boolean testExists( Flow flow, Hfs tap ) { try { // don't test for _SUCCESS if the tap is a file, only if a directory if( !tap.isDirectory( flow.getFlowProcess() ) ) return true; return new Hfs( new TextLine(), new Path( tap.getPath(), "_SUCCESS" ).toString() ).resourceExists( flow.getFlowProcess() ); } catch( IOException exception ) { throw new FlowException( exception ); } }
@Test public void testHfsAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "*" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
@Test public void testHfsAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "*" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
@Test public void testHfsBracketAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "{*}" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }
@Test public void testHfsBracketAsterisk() throws Exception { getPlatform().copyFromLocal( inputFileLower ); getPlatform().copyFromLocal( inputFileUpper ); Hfs sourceExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "{*}" ); assertTrue( sourceExists.resourceExists( getPlatform().getFlowProcess() ) ); TupleEntryIterator iterator = sourceExists.openForRead( getPlatform().getFlowProcess() ); assertTrue( iterator.hasNext() ); iterator.close(); try { Hfs sourceNotExists = new Hfs( new TextLine( new Fields( "offset", "line" ) ), InputData.inputPath + "/blah/" ); iterator = sourceNotExists.openForRead( getPlatform().getFlowProcess() ); fail(); } catch( IOException exception ) { // do nothing } }