protected boolean isSkipFlowStep() throws IOException { // if runID is not set, never skip a step if( flowStep.getFlow().getRunID() == null ) return false; return flowStep.allSourcesExist() && !flowStep.areSourcesNewer( flowStep.getSinkModified() ); }
protected void cleanIntermediateData( JobConf config, Tap sink ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } }
protected void cleanIntermediateData( JobConf config, Tap sink ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } }
@Override public void clean( TezConfiguration config ) { for( Tap sink : getSinkTaps() ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } } for( Tap tap : getTraps() ) cleanTapMetaData( config, tap ); }
@Override public void clean( TezConfiguration config ) { for( Tap sink : getSinkTaps() ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } } for( Tap tap : getTraps() ) cleanTapMetaData( config, tap ); }