private boolean isSimpleGlob() { if( Util.isEmpty( getHfs().getIdentifier() ) ) return false; return getHfs().getIdentifier().contains( "*" ); }
private boolean isSimpleGlob() { if( Util.isEmpty( getHfs().getIdentifier() ) ) return false; return getHfs().getIdentifier().contains( "*" ); }
@Override public String[] getChildIdentifiers( Configuration conf, int depth, boolean fullyQualified ) throws IOException { if( !resourceExists( conf ) ) return new String[ 0 ]; if( depth == 0 && !fullyQualified ) return new String[]{getIdentifier()}; String fullIdentifier = getFullIdentifier( conf ); int trim = fullyQualified ? 0 : fullIdentifier.length() + 1; Set<String> results = new LinkedHashSet<String>(); getChildPaths( conf, results, trim, new Path( fullIdentifier ), depth ); return results.toArray( new String[ results.size() ] ); }
private void getChildPaths( Configuration conf, Set<String> results, int trim, Path path, int depth ) throws IOException { if( depth == 0 ) { String substring = path.toString().substring( trim ); String identifier = getIdentifier(); if( identifier == null || identifier.isEmpty() ) results.add( new Path( substring ).toString() ); else results.add( new Path( identifier, substring ).toString() ); return; } FileStatus[] statuses = getFileSystem( conf ).listStatus( path, HIDDEN_FILES_FILTER ); if( statuses == null ) return; for( FileStatus fileStatus : statuses ) getChildPaths( conf, results, trim, fileStatus.getPath(), depth - 1 ); }
private void getChildPaths( Configuration conf, Set<String> results, int trim, Path path, int depth ) throws IOException { if( depth == 0 ) { String substring = path.toString().substring( trim ); String identifier = getIdentifier(); if( identifier == null || identifier.isEmpty() ) results.add( new Path( substring ).toString() ); else results.add( new Path( identifier, substring ).toString() ); return; } FileStatus[] statuses = getFileSystem( conf ).listStatus( path, HIDDEN_FILES_FILTER ); if( statuses == null ) return; for( FileStatus fileStatus : statuses ) getChildPaths( conf, results, trim, fileStatus.getPath(), depth - 1 ); }
@Override public String[] getChildIdentifiers( Configuration conf, int depth, boolean fullyQualified ) throws IOException { if( !resourceExists( conf ) ) return new String[ 0 ]; if( depth == 0 && !fullyQualified ) return new String[]{getIdentifier()}; String fullIdentifier = getFullIdentifier( conf ); int trim = fullyQualified ? 0 : fullIdentifier.length() + 1; Set<String> results = new LinkedHashSet<String>(); getChildPaths( conf, results, trim, new Path( fullIdentifier ), depth ); return results.toArray( new String[ results.size() ] ); }
public static String writeStateToDistCache( JobConf conf, String id, String kind, String stepState ) { if( Util.isEmpty( stepState ) ) return null; LOG.info( "writing step state to dist cache, too large for job conf, size: {}", stepState.length() ); String statePath = Hfs.getTempPath( conf ) + "/" + kind + "-state-" + id; Hfs temp = new Hfs( new TextLine(), statePath, SinkMode.REPLACE ); try { TupleEntryCollector writer = temp.openForWrite( new HadoopFlowProcess( conf ) ); writer.add( new Tuple( stepState ) ); writer.close(); } catch( IOException exception ) { throw new FlowException( "unable to write step state to Hadoop FS: " + temp.getIdentifier() ); } URI uri = new Path( statePath ).toUri(); DistributedCache.addCacheFile( uri, conf ); LOG.info( "using step state path: {}", uri ); return statePath; }
public static String writeStateToDistCache( JobConf conf, String id, String kind, String stepState ) { if( Util.isEmpty( stepState ) ) return null; LOG.info( "writing step state to dist cache, too large for job conf, size: {}", stepState.length() ); String statePath = Hfs.getTempPath( conf ) + "/" + kind + "-state-" + id; Hfs temp = new Hfs( new TextLine(), statePath, SinkMode.REPLACE ); try { TupleEntryCollector writer = temp.openForWrite( new HadoopFlowProcess( conf ) ); writer.add( new Tuple( stepState ) ); writer.close(); } catch( IOException exception ) { throw new FlowException( "unable to write step state to Hadoop FS: " + temp.getIdentifier() ); } URI uri = new Path( statePath ).toUri(); DistributedCache.addCacheFile( uri, conf ); LOG.info( "using step state path: {}", uri ); return statePath; }
throw new FlowException( "step state path is empty: " + temp.getIdentifier() ); throw new FlowException( "unable to find state path: " + temp.getIdentifier(), exception );
throw new FlowException( "step state path is empty: " + temp.getIdentifier() ); throw new FlowException( "unable to find state path: " + temp.getIdentifier(), exception );