@Override public boolean deleteResource(Object conf) throws IOException { return actualTap.deleteResource(conf); }
@Override public boolean deleteResource( Config conf ) throws IOException { return original.deleteResource( conf ); }
@Override public boolean deleteResource( Config conf ) throws IOException { return keepParentOnDelete || parent.deleteResource( conf ); }
@Override public boolean deleteResource( FlowProcess<? extends Config> flowProcess ) throws IOException { return original.deleteResource( flowProcess ); }
private boolean deleteParent( Properties conf ) throws IOException { return keepParentOnDelete || parent.deleteResource( conf ); }
@Override public boolean deleteResource(Object conf) throws IOException { return actualTap.deleteResource(conf); }
@Override public boolean deleteResource( FlowProcess<? extends TConfig> flowProcess ) throws IOException { return original.deleteResource( processProvider.apply( flowProcess ) ); }
@Override public boolean deleteResource( TConfig conf ) throws IOException { return original.deleteResource( configProvider.apply( conf ) ); }
/** * Method deleteResource deletes the resource represented by this instance. * * @param flowProcess of type FlowProcess * @return boolean * @throws IOException when the resource cannot be deleted */ public boolean deleteResource( FlowProcess<? extends Config> flowProcess ) throws IOException { return deleteResource( flowProcess.getConfig() ); }
@Override public boolean deleteResource( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.deleteResource( conf ) ) return false; } return true; }
/** * Method openForWrite opens the resource represented by this Tap instance for writing. * <p> * This method is for user application use and does honor the {@link SinkMode#REPLACE} settings. That is, if * SinkMode is set to {@link SinkMode#REPLACE} the underlying resource will be deleted. * <p> * Note if {@link SinkMode#UPDATE} is set, the resource will not be deleted. * * @param flowProcess of type FlowProcess * @return TupleEntryCollector * @throws java.io.IOException when the resource cannot be opened */ public TupleEntryCollector openForWrite( FlowProcess<? extends Config> flowProcess ) throws IOException { if( isReplace() ) deleteResource( flowProcess ); return openForWrite( flowProcess, null ); }
private void deleteOrFail( Tap tap ) throws IOException { if( !tap.resourceExists( getConfig() ) ) return; if( !tap.deleteResource( getConfig() ) ) throw new FlowTapException( "unable to delete resource: " + tap.getFullIdentifier( getFlowProcess() ) ); }
protected void cleanIntermediateData( JobConf config, Tap sink ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } }
protected void cleanIntermediateData( JobConf config, Tap sink ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } }
@Override public void clean( TezConfiguration config ) { for( Tap sink : getSinkTaps() ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } } for( Tap tap : getTraps() ) cleanTapMetaData( config, tap ); }
@Override public void clean( TezConfiguration config ) { for( Tap sink : getSinkTaps() ) { if( sink.isTemporary() && ( getFlow().getFlowStats().isSuccessful() || getFlow().getRunID() == null ) ) { try { sink.deleteResource( config ); } catch( Exception exception ) { // sink all exceptions, don't fail app logWarn( "unable to remove temporary file: " + sink, exception ); } } else { cleanTapMetaData( config, sink ); } } for( Tap tap : getTraps() ) cleanTapMetaData( config, tap ); }
@Test public void testSkipStrategiesKeep() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "keep" ), SinkMode.KEEP ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", new FlowSkipIfSinkExists().skipFlow( flow ) ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }
@Test public void testSkipStrategiesKeep() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "keep" ), SinkMode.KEEP ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", new FlowSkipIfSinkExists().skipFlow( flow ) ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }
@Test public void testSkipStrategiesReplace() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "replace" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); FlowSkipStrategy old = flow.getFlowSkipStrategy(); FlowSkipStrategy replaced = flow.setFlowSkipStrategy( new FlowSkipIfSinkExists() ); assertTrue( "not same instance", old == replaced ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }
@Test public void testSkipStrategiesReplace() throws Exception { getPlatform().copyFromLocal( inputFileApache ); Tap source = getPlatform().getTextFile( inputFileApache ); // !!! enable replace Tap sink = getPlatform().getTextFile( getOutputPath( "replace" ), SinkMode.REPLACE ); Pipe pipe = new Pipe( "test" ); pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) ); Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe ); sink.deleteResource( flow.getConfig() ); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); flow.complete(); assertTrue( "default skip", !flow.getFlowSkipStrategy().skipFlow( flow ) ); assertTrue( "exist skip", !new FlowSkipIfSinkExists().skipFlow( flow ) ); FlowSkipStrategy old = flow.getFlowSkipStrategy(); FlowSkipStrategy replaced = flow.setFlowSkipStrategy( new FlowSkipIfSinkExists() ); assertTrue( "not same instance", old == replaced ); validateLength( flow.openSource(), 10 ); // validate source, this once, as a sanity check validateLength( flow, 10, null ); }