@Override public boolean resourceExists(Object conf) throws IOException { return actualTap.resourceExists(conf); }
@Override public boolean resourceExists( Config conf ) throws IOException { return original.resourceExists( conf ); }
@Override public boolean resourceExists( FlowProcess<? extends Config> flowProcess ) throws IOException { return original.resourceExists( flowProcess ); }
@Override public boolean resourceExists( Config conf ) throws IOException { return parent.resourceExists( conf ); }
@Override public boolean resourceExists(Object conf) throws IOException { return actualTap.resourceExists(conf); }
@Override public boolean resourceExists( FlowProcess<? extends TConfig> flowProcess ) throws IOException { return original.resourceExists( processProvider.apply( flowProcess ) ); }
@Override public boolean resourceExists( TConfig conf ) throws IOException { return original.resourceExists( configProvider.apply( conf ) ); }
public boolean resourceExists( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.resourceExists( conf ) ) return false; } return true; }
boolean allSourcesExist() throws IOException { for( Tap tap : sources.keySet() ) { if( !tap.resourceExists( getConfig() ) ) return false; } return true; }
/** * Method resourceExists returns true if the path represented by this instance exists. * * @param flowProcess of type FlowProcess * @return true if the underlying resource already exists * @throws IOException when the status cannot be determined */ public boolean resourceExists( FlowProcess<? extends Config> flowProcess ) throws IOException { return resourceExists( flowProcess.getConfig() ); }
@Override public boolean resourceExists( Tap tap ) throws IOException { return tap.resourceExists( getConfig() ); }
@Override public boolean resourceExists( Config conf ) throws IOException { for( Tap tap : getTaps() ) { if( !tap.resourceExists( conf ) ) return false; } return true; }
public static long getSinkModified( Object config, Collection<Tap> sinks ) throws IOException { long sinkModified = Long.MAX_VALUE; for( Tap sink : sinks ) { if( sink.isReplace() || sink.isUpdate() ) sinkModified = -1L; else { if( !sink.resourceExists( config ) ) sinkModified = 0L; else sinkModified = Math.min( sinkModified, sink.getModifiedTime( config ) ); // return youngest mod date } } return sinkModified; }
public static long getSourceModified( Object confCopy, Iterator<Tap> values, long sinkModified ) throws IOException { long sourceModified = 0; while( values.hasNext() ) { Tap source = values.next(); if( source instanceof MultiSourceTap ) return getSourceModified( confCopy, ( (MultiSourceTap) source ).getChildTaps(), sinkModified ); sourceModified = source.getModifiedTime( confCopy ); // source modified returns zero if does not exist // this should minimize number of times we touch any file meta-data server if( sourceModified == 0 && !source.resourceExists( confCopy ) ) throw new FlowException( "source does not exist: " + source ); if( sinkModified < sourceModified ) return sourceModified; } return sourceModified; }
public String[] getChildIdentifiers( FileType<Config> fileType ) throws IOException { if( !( (Tap) fileType ).resourceExists( getSystemConfig() ) ) throw new IllegalStateException( "resource does not exist: " + ( (Tap) fileType ).getFullIdentifier( getSystemConfig() ) ); return fileType.getChildIdentifiers( getSystemConfig() ); }
private boolean resourceExistsAndNotEmpty( SchemaDef schemaDef, Tap tap ) { if( tap == null ) return false; try { Object configCopy = platformBroker.getFlowProcess().getConfigCopy(); if( !tap.resourceExists( configCopy ) ) return false; if( !( tap instanceof FileType ) ) return true; // is file not empty, or if directory, is any file not empty // just a safeguard against all files being empty return resourceNotEmpty( schemaDef, (FileType) tap, configCopy ); } catch( IOException exception ) { return false; } }
/** * Method deleteSinksIfReplace deletes all sinks that are configured with the {@link cascading.tap.SinkMode#REPLACE} flag. * * @throws IOException */ public void deleteSinksIfReplace() throws IOException { // verify all sinks before incrementally deleting for a replace for( Tap tap : sinks.values() ) { if( tap.isKeep() && tap.resourceExists( getConfig() ) ) throw new FlowTapException( "resource exists and sink mode is KEEP, cannot overwrite: " + tap.getFullIdentifier( getFlowProcess() ) ); } for( Tap tap : sinks.values() ) { if( tap.isReplace() ) deleteOrFail( tap ); } }
private void deleteOrFail( Tap tap ) throws IOException { if( !tap.resourceExists( getConfig() ) ) return; if( !tap.deleteResource( getConfig() ) ) throw new FlowTapException( "unable to delete resource: " + tap.getFullIdentifier( getFlowProcess() ) ); }
private double getRowsInternal() { PlatformBroker platformBroker = getPlatformBroker(); if( platformBroker == null ) return super.getRows(); SchemaCatalogManager catalog = platformBroker.getCatalogManager(); if( catalog == null ) return super.getRows(); TableDef tableDef = getTapTable().getTableDef(); try { Tap tap = catalog.createTapFor( tableDef, SinkMode.KEEP ); if( tap != null ) { if( !tap.resourceExists( platformBroker.getSystemConfig() ) ) return 0.0; else if( tap instanceof FileType ) return ( (FileType) tap ).getSize( platformBroker.getSystemConfig() ); // actually returns bytes } } catch( Exception exception ) { LOG.warn( "unable to create tap for: " + tableDef ); } return super.getRows(); } }
@Test public void testSkippedCascade() throws IOException { getPlatform().copyFromLocal( inputFileIps ); String path = "skipped"; Flow first = firstFlow( path + "/first", false ); Flow second = secondFlow( first.getSink(), path + "/second" ); Flow third = thirdFlow( second.getSink(), path + "/third" ); Flow fourth = fourthFlow( third.getSink(), path + "/fourth" ); CountingFlowListener flowListener = new CountingFlowListener(); second.addListener( flowListener ); Cascade cascade = new CascadeConnector( getProperties() ).connect( first, second, third, fourth ); cascade.setFlowSkipStrategy( new FlowSkipStrategy() { public boolean skipFlow( Flow flow ) throws IOException { return true; } } ); cascade.start(); cascade.complete(); assertEquals( 1, flowListener.skipped ); assertFalse( "file exists", fourth.getSink().resourceExists( fourth.getConfig() ) ); }