jobEntry.setName( wName.getText() ); jobEntry.setCopyEmptyFolders( wCopyEmptyFolders.getSelection() ); jobEntry.setoverwrite_files( wOverwriteFiles.getSelection() ); jobEntry.setIncludeSubfolders( wIncludeSubfolders.getSelection() ); jobEntry.setArgFromPrevious( wPrevious.getSelection() ); jobEntry.setRemoveSourceFiles( wRemoveSourceFiles.getSelection() ); jobEntry.setAddresultfilesname( wAddFileToResult.getSelection() ); jobEntry.setDestinationIsAFile( wDestinationIsAFile.getSelection() ); jobEntry.setCreateDestinationFolder( wCreateDestinationFolder.getSelection() ); source = JobEntryCopyFiles.SOURCE_URL + i + "-" + source; dest = JobEntryCopyFiles.DEST_URL + i + "-" + dest; jobEntry.source_filefolder[i] = jobEntry.loadURL( source, sourceNc, getMetaStore(), sourceDestinationMappings ); jobEntry.destination_filefolder[i] = jobEntry.loadURL( dest, destNc, getMetaStore(), sourceDestinationMappings ); jobEntry.wildcard[i] = wild; jobEntry.setConfigurationMappings( sourceDestinationMappings );
if ( jobEntry.getName() != null ) { wName.setText( jobEntry.getName() ); if ( jobEntry.source_filefolder[i] != null ) { String sourceUrl = jobEntry.source_filefolder[i]; String clusterName = jobEntry.getConfigurationBy( sourceUrl ); if ( clusterName != null ) { clusterName = sourceUrl = clusterName.equals( LOCAL_ENVIRONMENT ) || clusterName.equals( STATIC_ENVIRONMENT ) ? sourceUrl : jobEntry.getUrlPath( sourceUrl ); String clusterName = jobEntry.getConfigurationBy( destinationURL ); if ( clusterName != null ) { clusterName = clusterName.startsWith( JobEntryCopyFiles.LOCAL_DEST_FILE ) ? LOCAL_ENVIRONMENT : clusterName; destinationURL = clusterName.equals( LOCAL_ENVIRONMENT ) || clusterName.equals( STATIC_ENVIRONMENT ) ? destinationURL : jobEntry.getUrlPath( destinationURL );
String[] vsourcefilefolder = preprocessfilefilder( source_filefolder ); String[] vdestinationfilefolder = preprocessfilefilder( destination_filefolder ); if ( source_filefolder != null ) { for ( int i = 0; i < source_filefolder.length; i++ ) { retval.append( " <field>" ).append( Const.CR ); saveSource( retval, source_filefolder[i] ); saveDestination( retval, destination_filefolder[i] ); if ( parentJobMeta != null ) { parentJobMeta.getNamedClusterEmbedManager().registerUrl( vsourcefilefolder[i] );
public JobEntryCopyFilesDialog( Shell parent, JobEntryInterface jobEntryInt, Repository rep, JobMeta jobMeta ) { super( parent, jobEntryInt, rep, jobMeta ); jobEntry = (JobEntryCopyFiles) jobEntryInt; if ( this.jobEntry.getName() == null ) { this.jobEntry.setName( BaseMessages.getString( PKG, "JobCopyFiles.Name.Default" ) ); } }
protected String loadSourceRep( Repository rep, ObjectId id_jobentry, int a ) throws KettleException { String source_filefolder = rep.getJobEntryAttributeString( id_jobentry, a, SOURCE_FILE_FOLDER ); String ncName = rep.getJobEntryAttributeString( id_jobentry, a, SOURCE_CONFIGURATION_NAME ); return loadURL( source_filefolder, ncName, getMetaStore(), configurationMappings ); }
if ( isBasic() ) { logBasic( BaseMessages.getString( PKG, "JobCopyFiles.Log.Starting" ) ); String[] vsourcefilefolder = preprocessfilefilder( source_filefolder ); String[] vdestinationfilefolder = preprocessfilefilder( destination_filefolder ); String[] vwildcard = wildcard; if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobCopyFiles.Log.ArgFromPrevious.Found", ( rows != null ? rows .size() : 0 ) + "" ) ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobCopyFiles.Log.ProcessingRow", KettleVFS.getFriendlyURI( environmentSubstitute( vsourcefilefolder_previous ) ), KettleVFS.getFriendlyURI( environmentSubstitute( vdestinationfilefolder_previous ) ), vwildcard_previous ) ); if ( !processFileFolder( vsourcefilefolder_previous, vdestinationfilefolder_previous, vwildcard_previous, parentJob, result ) ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "JobCopyFiles.Log.IgnoringRow", KettleVFS.getFriendlyURI( environmentSubstitute( vsourcefilefolder[ iteration ] ) ), KettleVFS.getFriendlyURI( environmentSubstitute( vdestinationfilefolder[ iteration ] ) ), vwildcard[ iteration ] ) ); if ( isBasic() ) { logBasic( BaseMessages.getString( PKG, "JobCopyFiles.Log.ProcessingRow",
@Test public void fileCopied() throws Exception { String srcPath = "path/to/file"; String destPath = "path/to/dir"; entry.source_filefolder = new String[] { srcPath }; entry.destination_filefolder = new String[] { destPath }; entry.wildcard = new String[] { EMPTY }; Result result = entry.execute( new Result(), 0 ); verify( entry ).processFileFolder( anyString(), anyString(), anyString(), any( Job.class ), any( Result.class ) ); verify( entry, atLeast( 1 ) ).preprocessfilefilder( any( String[].class ) ); assertFalse( result.getResult() ); assertEquals( 1, result.getNrErrors() ); verify( mockNamedClusterEmbedManager ).passEmbeddedMetastoreKey( anyObject(), anyString() ); }
@Test public void fileNotCopied() throws Exception { entry.source_filefolder = new String[] { EMPTY }; entry.destination_filefolder = new String[] { EMPTY }; entry.wildcard = new String[] { EMPTY }; entry.execute( new Result(), 0 ); verify( entry, never() ).processFileFolder( anyString(), anyString(), anyString(), any( Job.class ), any( Result.class ) ); }
public void check( List<CheckResultInterface> remarks, JobMeta jobMeta, VariableSpace space, Repository repository, IMetaStore metaStore ) { boolean res = JobEntryValidatorUtils.andValidator().validate( this, "arguments", remarks, AndValidator.putValidators( JobEntryValidatorUtils.notNullValidator() ) ); if ( !res ) { return; } ValidatorContext ctx = new ValidatorContext(); AbstractFileValidator.putVariableSpace( ctx, getVariables() ); AndValidator.putValidators( ctx, JobEntryValidatorUtils.notNullValidator(), JobEntryValidatorUtils.fileExistsValidator() ); for ( int i = 0; i < source_filefolder.length; i++ ) { JobEntryValidatorUtils.andValidator().validate( this, "arguments[" + i + "]", remarks, ctx ); } }
wFields.getActiveTableItem().setText( wFields.getActiveTableColumn() - 1, S3_ENVIRONMENT ); } else if ( namedClusterWidget != null && namedClusterWidget.getSelectedNamedCluster() != null ) { url = jobEntry.getUrlPath( url ); wFields.getActiveTableItem().setText( wFields.getActiveTableColumn() - 1, namedClusterWidget.getSelectedNamedCluster().getName() );
protected String loadSource( Node fnode ) { String source_filefolder = XMLHandler.getTagValue( fnode, SOURCE_FILE_FOLDER ); String ncName = XMLHandler.getTagValue( fnode, SOURCE_CONFIGURATION_NAME ); return loadURL( source_filefolder, ncName, getMetaStore(), configurationMappings ); }
public JobEntryHadoopCopyFilesDialog( Shell parent, JobEntryInterface jobEntryInt, Repository rep, JobMeta jobMeta ) { super( parent, jobEntryInt, rep, jobMeta ); jobEntry = (JobEntryCopyFiles) jobEntryInt; jobEntryHadoopCopyFiles = (JobEntryHadoopCopyFiles) jobEntry; namedClusterService = jobEntryHadoopCopyFiles.getNamedClusterService(); runtimeTestActionService = jobEntryHadoopCopyFiles.getRuntimeTestActionService(); runtimeTester = jobEntryHadoopCopyFiles.getRuntimeTester(); if ( this.jobEntry.getName() == null ) { this.jobEntry.setName( BaseMessages.getString( BASE_PKG, "JobCopyFiles.Name.Default" ) ); } }
@Test public void filesCopied() throws Exception { String[] srcPath = new String[] { "path1", "path2", "path3" }; String[] destPath = new String[] { "dest1", "dest2", "dest3" }; entry.source_filefolder = srcPath; entry.destination_filefolder = destPath; entry.wildcard = new String[] { EMPTY, EMPTY, EMPTY }; Result result = entry.execute( new Result(), 0 ); verify( entry, times( srcPath.length ) ).processFileFolder( anyString(), anyString(), anyString(), any( Job.class ), any( Result.class ) ); assertFalse( result.getResult() ); assertEquals( 3, result.getNrErrors() ); }
@VisibleForTesting @Override protected VariableSpace getVariables() { return super.getVariables(); }
jobEntry.setName( wName.getText() ); jobEntry.setCopyEmptyFolders( wCopyEmptyFolders.getSelection() ); jobEntry.setoverwrite_files( wOverwriteFiles.getSelection() ); jobEntry.setIncludeSubfolders( wIncludeSubfolders.getSelection() ); jobEntry.setArgFromPrevious( wPrevious.getSelection() ); jobEntry.setRemoveSourceFiles( wRemoveSourceFiles.getSelection() ); jobEntry.setAddresultfilesname( wAddFileToResult.getSelection() ); jobEntry.setDestinationIsAFile( wDestinationIsAFile.getSelection() ); jobEntry.setCreateDestinationFolder( wCreateDestinationFolder.getSelection() ); dest = JobEntryCopyFiles.DEST_URL + i + "-" + dest; jobEntry.source_filefolder[i] = jobEntry.loadURL( source, sourceNc, getMetaStore(), namedClusterURLMappings ); jobEntry.destination_filefolder[i] = jobEntry.loadURL( dest, destNc, getMetaStore(), namedClusterURLMappings ); jobEntry.wildcard[i] = wild; jobEntry.setConfigurationMappings( namedClusterURLMappings ); dispose();
if ( jobEntry.getName() != null ) { wName.setText( jobEntry.getName() ); if ( jobEntry.source_filefolder[i] != null ) { String sourceUrl = jobEntry.source_filefolder[i]; String clusterName = jobEntry.getConfigurationBy( sourceUrl ); if ( clusterName != null ) { clusterName = sourceUrl = clusterName.equals( LOCAL_ENVIRONMENT ) || clusterName.equals( STATIC_ENVIRONMENT ) || clusterName.equals( S3_ENVIRONMENT ) ? sourceUrl : jobEntry.getUrlPath( sourceUrl.replace( JobEntryCopyFiles.SOURCE_URL + i + "-", "" ) ); String clusterName = jobEntry.getConfigurationBy( destinationURL ); if ( clusterName != null ) { clusterName = destinationURL = clusterName.equals( LOCAL_ENVIRONMENT ) || clusterName.equals( STATIC_ENVIRONMENT ) || clusterName.equals( S3_ENVIRONMENT ) ? destinationURL : jobEntry.getUrlPath( destinationURL.replace( JobEntryCopyFiles.DEST_URL + i + "-", "" ) );
protected String loadDestination( Node fnode ) { String destination_filefolder = XMLHandler.getTagValue( fnode, DESTINATION_FILE_FOLDER ); String ncName = XMLHandler.getTagValue( fnode, DESTINATION_CONFIGURATION_NAME ); return loadURL( destination_filefolder, ncName, getMetaStore(), configurationMappings ); }
protected String loadDestinationRep( Repository rep, ObjectId id_jobentry, int a ) throws KettleException { String destination_filefolder = rep.getJobEntryAttributeString( id_jobentry, a, DESTINATION_FILE_FOLDER ); String ncName = rep.getJobEntryAttributeString( id_jobentry, a, DESTINATION_CONFIGURATION_NAME ); return loadURL( destination_filefolder, ncName, getMetaStore(), configurationMappings ); }