private void submitExecutionScript(StorageClient pSms) throws Exception { LOGGER.info("Submit job execution script."); InputStream source = UnicoreClient.class.getResourceAsStream(JOB_EXECUTION_SCRIPT); RByteIOClient fileClient = pSms.getImport("/" + JOB_EXECUTION_SCRIPT); fileClient.writeAllData(source); }
private void configure(FileTransferClient ftc, Map<String,String>params){ if(ftc instanceof UFTPFileTransferClient){ UFTPFileTransferClient u=(UFTPFileTransferClient)ftc; String secret=params.get(UFTPConstants.PARAM_SECRET); u.setSecret(secret); } }
/** * if possible, copy the local executable flag to the remote file * @param sourceFile - local file * @throws Exception */ private void copyProperties(File sourceFile, StorageClient sms, String target)throws Exception{ boolean x=sourceFile.canExecute(); try{ if(x){ sms.changePermissions(target, true, true, x); } }catch(Exception ex){ // System.out.println("Can't set exectuable flag on remote file.",ex); } }
registryEpr.addNewAddress().setStringValue(registry); LOGGER.info("Connect UNCIORE Registry Service at '" + registryEpr.getAddress().getStringValue() + "'."); RegistryClient registry = new RegistryClient(registryEpr.getAddress().getStringValue(), registryEpr, pSecurityProperties); List<EndpointReferenceType> tsfList = registry.listServices(TargetSystemFactory.TSF_PORT); for (EndpointReferenceType tsfEpr : tsfList) TSFClient tsf = new TSFClient(tsfEpr.getAddress().getStringValue(), tsfEpr, pSecurityProperties); List<EndpointReferenceType> tssList = tsf.getTargetSystems(); for (EndpointReferenceType tssEpr : tssList) TSSClient tss = tsf.createTSS(in); LOGGER.info("Create Target System Service (TSS) at '" + tss.getEPR().getAddress().getStringValue() + "'.");
/** * @param pGridInput * @return * @throws Exception */ private DistributedAlgorithmOutput run(DistributedAlgorithmInput pGridInput) throws Exception { // submit job JobClient job = submitJob(targetSystem); // stage in input files StorageClient smsUspace = job.getUspaceClient(); submitInputFiles(smsUspace, pGridInput); // stage in execution script submitExecutionScript(smsUspace); // run job LOGGER.info("Run job."); job.waitUntilReady(WAIT_UNTIL_READY_TIMEOUT); job.start(); job.waitUntilDone(WAIT_UNTIL_DONE_TIMEOUT); // fetch process outcome DistributedAlgorithmOutput processOutput = getAlgorithmOutput(smsUspace); // destroy job at target system job.destroy(); return processOutput; }
private StorageClient getHomeStorageClient(TSSClient tss, IUASSecurityProperties pSecurityProperties) { List<EndpointReferenceType> storageList = tss.getStorages(); if (storageList.size() < 1) { LOGGER.error("No home storage found at target system '" + tss.getTargetSystemName() + "'."); throw new RuntimeException("No home storage found at target system '" + tss.getTargetSystemName() + "'."); } if (storageList.size() > 1) { LOGGER.error("More than one home storage found at target system '" + tss.getTargetSystemName() + "'."); throw new RuntimeException("More than one home storage found at target system '" + tss.getTargetSystemName() + "'."); } EndpointReferenceType smsEpr = storageList.get(0); LOGGER.info("Create Storage Management Service (SMS) at '" + smsEpr.getAddress().getStringValue() + "'."); StorageClient client; try { client = new StorageClient(smsEpr.getAddress().getStringValue(), smsEpr, pSecurityProperties); } catch (Exception e) { LOGGER.error("Error during creation of Storage Management Service (SMS)."); throw new RuntimeException("Error during creation of Storage Management Service (SMS)."); } return client; }
/** * @param pSms * @param pAlgorithmInput * @param tc * @throws Exception */ private void submitInputFiles(StorageClient pSms, DistributedAlgorithmInput pAlgorithmInput) throws Exception { // create compressed input data LOGGER.info("Serialize input data."); byte[] data = CompressUtilities.serialize(pAlgorithmInput); byte[] dataCompressed = CompressUtilities.createCompressedData(data); LOGGER.info("Compress input data (ratio: " + ((double) dataCompressed.length) / ((double) data.length) + ")."); // submit input data LOGGER.info("Submit input data."); RByteIOClient fileClient = pSms.getImport("/" + TARGET_SYSTEM_INPUT_FILE_NAME); fileClient.write(dataCompressed); }
protected byte[] stageOut(final StorageClient uspace, final String pFileName) throws ExceptionReport { try { RByteIOClient fileClient = uspace.getExport(pFileName); ByteArrayOutputStream bos = new ByteArrayOutputStream(); fileClient.readAllData(bos); return bos.toByteArray(); } catch (IOException e) { LOGGER.error("Error while fetching remote file '" + pFileName + "'."); throw new ExceptionReport("Error while fetching remote file '" + pFileName + "'.", ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } }
protected JobClient submitJob(final TSSClient tss) throws Exception { // create job definition document LOGGER.info("Create job definition document."); JobDefinitionDocument definition = getJobDefinition(); // submit job definition document LOGGER.info("Submit job definition document."); SubmitDocument submit = SubmitDocument.Factory.newInstance(); submit.addNewSubmit().setJobDefinition(definition.getJobDefinition()); SubmitResponseDocument response = tss.Submit(submit); // create job client EndpointReferenceType jobEpr = response.getSubmitResponse().getJobReference(); LOGGER.info("Create job at '" + jobEpr.getAddress().getStringValue() + "'."); return new JobClient(jobEpr.getAddress().getStringValue(), jobEpr, securityProperties); }
/** * upload a set of files to a remote directory (which must exist) * * @param files * @param remoteDirectory * @param sms * @param protocol * @param extraParameters * @param msg * @throws Exception */ private void uploadFiles(File[]files, String remoteDirectory, StorageClient sms, ProtocolType.Enum protocol, Map<String,String>extraParameters)throws Exception{ for(File localFile: files){ String target=remoteDirectory+"/"+localFile.getName(); if(localFile.isDirectory()){ if(!recurse){ System.out.println("Skipping directory "+localFile.getAbsolutePath()); }else{ File[] fileset=localFile.listFiles(); sms.createDirectory(target); uploadFiles(fileset,target,sms,protocol,extraParameters); } }else{ uploadFile(localFile,target,sms,protocol,extraParameters); } } }
/** * check if the given path denotes a valid remote directory * @param remotePath - the path * @param sms - the storage * @return <code>true</code> if the remote directory exists and is a directory */ protected boolean isValidDirectory(String remotePath, StorageClient sms){ boolean result=false; if(! ("/".equals(remotePath) || ".".equals(remotePath)) ){ try{ GridFileType gft=sms.listProperties(remotePath); result=gft.getIsDirectory(); }catch(Exception ex){ result=false; } } else result=true; return result; }
protected void performWildCardExport(StorageClient sms)throws Exception{ String dir=getDir(from); if(dir==null)dir="/"; GridFileType[] files=sms.find(dir, false, from, false, null, null); File targetDir=targetStream==null?new File(to):null; if(targetStream==null){ if(!targetDir.isDirectory())throw new IOException("Target is not a directory."); } for(GridFileType f: files){ download(f, targetDir, sms); } }
/** * @param tss * @return * @throws Exception */ private StorageClient getHomeStorageClient(TSSClient tss, IUASSecurityProperties pSecurityProperties) throws ExceptionReport { try { List<EndpointReferenceType> storageList = tss.getStorages(); if (storageList.size() < 1) { LOGGER.error("No home storage found at target system '" + tss.getTargetSystemName() + "'."); throw new Exception("No home storage found at target system '" + tss.getTargetSystemName() + "'."); } if (storageList.size() > 1) { LOGGER.warn("More than one home storage found at target system '" + tss.getTargetSystemName() + "'."); } EndpointReferenceType smsEpr = storageList.get(0); LOGGER.info("Select Storage Management Service (SMS) at '" + smsEpr.getAddress().getStringValue() + "'."); StorageClient client = new StorageClient(smsEpr.getAddress().getStringValue(), smsEpr, pSecurityProperties); return client; } catch (Exception e) { LOGGER.error("Error while accessing home storage."); throw new ExceptionReport("Error while accessing home storage.", ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } }
/** * @param pSms * @param tc * @throws Exception */ private void submitExecutionScript(StorageClient pSms) throws Exception { LOGGER.info("Submit job execution script."); InputStream source = Unicore6Client.class.getResourceAsStream(JOB_EXECUTION_SCRIPT); RByteIOClient fileClient = pSms.getImport("/" + JOB_EXECUTION_SCRIPT); fileClient.writeAllData(source); }
protected void submitInputData(StorageClient pSms, UnicoreAlgorithmInput pAlgorithmInput) throws Exception { RByteIOClient fileClient = pSms.getImport("/" + TARGET_SYSTEM_INPUT_FILE_NAME); // create serialized input data LOGGER.info("Serialize input data."); byte[] data = Compression.toByteArray(pAlgorithmInput); // create input data Properties unicoreProperties = UnicoreAlgorithmRepository.getInstance().getUnicoreProperties(); byte[] dataCompressed = null; if (Boolean.parseBoolean(unicoreProperties.getProperty(UnicoreAlgorithmRepository.CFG_COMPRESSION))) { dataCompressed = Compression.createCompressedData(data, Boolean.parseBoolean(unicoreProperties .getProperty(UnicoreAlgorithmRepository.CFG_COMPRESSION))); LOGGER.info("Compress input data (ratio: " + ((double) dataCompressed.length) / ((double) data.length) + ")."); } // submit input data if (Boolean.parseBoolean(unicoreProperties.getProperty(UnicoreAlgorithmRepository.CFG_COMPRESSION))) { LOGGER.info("Submit compressed input data."); fileClient.write(dataCompressed); } else { LOGGER.info("Submit input data."); fileClient.write(data); } }
/** * @param uspace * @param pFileName * @return * @throws IOException * @throws Exception */ protected byte[] stageOut(final StorageClient uspace, final String pFileName) throws ExceptionReport { try { RByteIOClient fileClient = uspace.getExport(pFileName); ByteArrayOutputStream bos = new ByteArrayOutputStream(); fileClient.readAllData(bos); return bos.toByteArray(); } catch (IOException e) { LOGGER.error("Error while fetching remote file '" + pFileName + "'."); throw new ExceptionReport("Error while fetching remote file '" + pFileName + "'.", ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } } }
/** * @param tss * @param tc * @return * @throws Exception */ private JobClient submitJob(final TSSClient tss) throws Exception { // create job definition document LOGGER.info("Create job definition document."); JobDefinitionDocument definition = getJobDefinition(); // submit job definition document LOGGER.info("Submit job definition document."); SubmitDocument submit = SubmitDocument.Factory.newInstance(); submit.addNewSubmit().setJobDefinition(definition.getJobDefinition()); SubmitResponseDocument response = tss.Submit(submit); // create job client EndpointReferenceType jobEpr = response.getSubmitResponse().getJobReference(); LOGGER.info("Create job at '" + jobEpr.getAddress().getStringValue() + "'."); return new JobClient(jobEpr.getAddress().getStringValue(), jobEpr, securityProperties); }
private void configure(FileTransferClient ftc, Map<String,String>params){ if(ftc instanceof UFTPFileTransferClient){ UFTPFileTransferClient u=(UFTPFileTransferClient)ftc; String secret=params.get(UFTPConstants.PARAM_SECRET); u.setSecret(secret); } } }
public void perform(StorageClient sms)throws Exception{ boolean isWildcard=hasWildCards(from); boolean isDirectory=false; GridFileType gridSource=null; if(isWildcard){ performWildCardExport(sms); } else { //check if source is a directory gridSource=sms.listProperties(from); isDirectory=gridSource.getIsDirectory(); if(isDirectory){ if(forceFileOnly){ throw new IOException("Source is a directory"); } performDirectoryExport(gridSource, new File(to), sms); } else{ download(gridSource,new File(to),sms); } } }
protected void submitWpsConfiguration(StorageClient pSms) throws Exception { RByteIOClient fileClient = pSms.getImport("/wps_config.xml"); File f = new File(WPSConfig.getConfigPath()); FileInputStream is = new FileInputStream(f); fileClient.writeAllData(is); is.close(); } }