protected byte[] stageOut(final StorageClient uspace, final String pFileName) throws ExceptionReport { try { RByteIOClient fileClient = uspace.getExport(pFileName); ByteArrayOutputStream bos = new ByteArrayOutputStream(); fileClient.readAllData(bos); return bos.toByteArray(); } catch (IOException e) { LOGGER.error("Error while fetching remote file '" + pFileName + "'."); throw new ExceptionReport("Error while fetching remote file '" + pFileName + "'.", ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } }
private void submitExecutionScript(StorageClient pSms) throws Exception { LOGGER.info("Submit job execution script."); InputStream source = UnicoreClient.class.getResourceAsStream(JOB_EXECUTION_SCRIPT); RByteIOClient fileClient = pSms.getImport("/" + JOB_EXECUTION_SCRIPT); fileClient.writeAllData(source); }
/** * upload a set of files to a remote directory (which must exist) * * @param files * @param remoteDirectory * @param sms * @param protocol * @param extraParameters * @param msg * @throws Exception */ private void uploadFiles(File[]files, String remoteDirectory, StorageClient sms, ProtocolType.Enum protocol, Map<String,String>extraParameters)throws Exception{ for(File localFile: files){ String target=remoteDirectory+"/"+localFile.getName(); if(localFile.isDirectory()){ if(!recurse){ System.out.println("Skipping directory "+localFile.getAbsolutePath()); }else{ File[] fileset=localFile.listFiles(); sms.createDirectory(target); uploadFiles(fileset,target,sms,protocol,extraParameters); } }else{ uploadFile(localFile,target,sms,protocol,extraParameters); } } }
try directoryList = smsHome.listDirectory("."); smsHome.createDirectory("./lib"); directoryList = smsHome.listDirectory("./lib"); out.addNewImportFile().setProtocol(ProtocolType.RBYTEIO); out.getImportFile().setDestination("./lib/" + applicationFile); ImportFileResponseDocument outDoc = smsHome.ImportFile(out); EndpointReferenceType outEpr = outDoc.getImportFileResponse().getImportEPR(); RByteIOClient fileClient = new RByteIOClient(outEpr.getAddress().getStringValue(), outEpr, pSecurityProperties);
return; System.out.println("Downloading remote file '"+sms.getUrl()+"#/"+path+"' -> "+localFile.getAbsolutePath()); os=new FileOutputStream(localFile.getAbsolutePath(), mode.equals(Mode.append)); chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()])); Map<String,String>extraParameters=makeExtraParameters(chosenProtocol); ftc=sms.getExport(path,extraParameters,chosenProtocol); configure(ftc, extraParameters); System.out.println("DEB:File transfer URL : "+ftc.getUrl());
try directoryList = smsHome.listDirectory("."); out.addNewImportFile().setProtocol(ProtocolType.RBYTEIO); out.getImportFile().setDestination("./" + applicationFile); ImportFileResponseDocument outDoc = smsHome.ImportFile(out); EndpointReferenceType outEpr = outDoc.getImportFileResponse().getImportEPR(); RByteIOClient fileClient = new RByteIOClient(outEpr.getAddress().getStringValue(), outEpr, pSecurityProperties);
remotePath+=localFile.getName(); System.out.println("Uploading local file '"+localFile.getAbsolutePath()+"' -> '"+sms.getUrl()+"#"+remotePath+"'"); is=new FileInputStream(localFile.getAbsolutePath()); boolean append=Mode.append.equals(mode); ftc=sms.getImport(remotePath, append, extraParameters, protocol); configure(ftc, extraParameters); if(append)ftc.setAppend(true);
private StorageClient getHomeStorageClient(TSSClient tss, IUASSecurityProperties pSecurityProperties) { List<EndpointReferenceType> storageList = tss.getStorages(); if (storageList.size() < 1) { LOGGER.error("No home storage found at target system '" + tss.getTargetSystemName() + "'."); throw new RuntimeException("No home storage found at target system '" + tss.getTargetSystemName() + "'."); } if (storageList.size() > 1) { LOGGER.error("More than one home storage found at target system '" + tss.getTargetSystemName() + "'."); throw new RuntimeException("More than one home storage found at target system '" + tss.getTargetSystemName() + "'."); } EndpointReferenceType smsEpr = storageList.get(0); LOGGER.info("Create Storage Management Service (SMS) at '" + smsEpr.getAddress().getStringValue() + "'."); StorageClient client; try { client = new StorageClient(smsEpr.getAddress().getStringValue(), smsEpr, pSecurityProperties); } catch (Exception e) { LOGGER.error("Error during creation of Storage Management Service (SMS)."); throw new RuntimeException("Error during creation of Storage Management Service (SMS)."); } return client; }
throw new GFacHandlerException("Cannot create storage..", e2); jobExecutionContext.setProperty(PROP_SMS_EPR, storageClient.getEPR()); } else { EndpointReferenceType eprt = (EndpointReferenceType) jobExecutionContext.getProperty(PROP_SMS_EPR); try { storageClient = new StorageClient(eprt, secProperties); } catch (Exception e) { throw new GFacHandlerException("Cannot create storage..", e);
chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()])); Map<String,String>extraParameters=makeExtraParameters(chosenProtocol); sms.createDirectory(target); uploadFiles(fileset,target,sms,chosenProtocol,extraParameters);
protected void performDirectoryExport(GridFileType directory, File targetDirectory, StorageClient sms)throws Exception{ if(!targetDirectory.exists()|| !targetDirectory.canWrite()){ throw new IOException("Target directory <"+to+"> does not exist or is not writable!"); } if(!targetDirectory.isDirectory()){ throw new IOException("Target <"+to+"> is not a directory!"); } GridFileType[]gridFiles=sms.listDirectory(directory.getPath()); for(GridFileType file: gridFiles){ if(file.getIsDirectory()){ if(!recurse) { System.out.println("Skipping directory "+file.getPath()); continue; } else{ File newTargetDirectory=new File(targetDirectory,getName(file.getPath())); boolean success=newTargetDirectory.mkdirs(); if(!success)throw new IOException("Can create directory: "+newTargetDirectory.getAbsolutePath()); performDirectoryExport(file, newTargetDirectory, sms); continue; } } download(file, new File(targetDirectory,getName(file.getPath())), sms); } }
/** * if possible, copy the local executable flag to the remote file * @param sourceFile - local file * @throws Exception */ private void copyProperties(File sourceFile, StorageClient sms, String target)throws Exception{ boolean x=sourceFile.canExecute(); try{ if(x){ sms.changePermissions(target, true, true, x); } }catch(Exception ex){ // System.out.println("Can't set exectuable flag on remote file.",ex); } }
protected void performWildCardExport(StorageClient sms)throws Exception{ String dir=getDir(from); if(dir==null)dir="/"; GridFileType[] files=sms.find(dir, false, from, false, null, null); File targetDir=targetStream==null?new File(to):null; if(targetStream==null){ if(!targetDir.isDirectory())throw new IOException("Target is not a directory."); } for(GridFileType f: files){ download(f, targetDir, sms); } }
try { if (storageClient != null) { storageClient.destroy();
/** * @param tss * @return * @throws Exception */ private StorageClient getHomeStorageClient(TSSClient tss, IUASSecurityProperties pSecurityProperties) throws ExceptionReport { try { List<EndpointReferenceType> storageList = tss.getStorages(); if (storageList.size() < 1) { LOGGER.error("No home storage found at target system '" + tss.getTargetSystemName() + "'."); throw new Exception("No home storage found at target system '" + tss.getTargetSystemName() + "'."); } if (storageList.size() > 1) { LOGGER.warn("More than one home storage found at target system '" + tss.getTargetSystemName() + "'."); } EndpointReferenceType smsEpr = storageList.get(0); LOGGER.info("Select Storage Management Service (SMS) at '" + smsEpr.getAddress().getStringValue() + "'."); StorageClient client = new StorageClient(smsEpr.getAddress().getStringValue(), smsEpr, pSecurityProperties); return client; } catch (Exception e) { LOGGER.error("Error while accessing home storage."); throw new ExceptionReport("Error while accessing home storage.", ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } }
/** * @param pSms * @param tc * @throws Exception */ private void submitExecutionScript(StorageClient pSms) throws Exception { LOGGER.info("Submit job execution script."); InputStream source = Unicore6Client.class.getResourceAsStream(JOB_EXECUTION_SCRIPT); RByteIOClient fileClient = pSms.getImport("/" + JOB_EXECUTION_SCRIPT); fileClient.writeAllData(source); }
/** * @param uspace * @param pFileName * @return * @throws IOException * @throws Exception */ protected byte[] stageOut(final StorageClient uspace, final String pFileName) throws ExceptionReport { try { RByteIOClient fileClient = uspace.getExport(pFileName); ByteArrayOutputStream bos = new ByteArrayOutputStream(); fileClient.readAllData(bos); return bos.toByteArray(); } catch (IOException e) { LOGGER.error("Error while fetching remote file '" + pFileName + "'."); throw new ExceptionReport("Error while fetching remote file '" + pFileName + "'.", ExceptionReport.REMOTE_COMPUTATION_ERROR, e); } } }
protected void submitWpsConfiguration(StorageClient pSms) throws Exception { RByteIOClient fileClient = pSms.getImport("/wps_config.xml"); File f = new File(WPSConfig.getConfigPath()); FileInputStream is = new FileInputStream(f); fileClient.writeAllData(is); is.close(); } }
/** * @param pSms * @param pAlgorithmInput * @param tc * @throws Exception */ private void submitInputFiles(StorageClient pSms, DistributedAlgorithmInput pAlgorithmInput) throws Exception { // create compressed input data LOGGER.info("Serialize input data."); byte[] data = CompressUtilities.serialize(pAlgorithmInput); byte[] dataCompressed = CompressUtilities.createCompressedData(data); LOGGER.info("Compress input data (ratio: " + ((double) dataCompressed.length) / ((double) data.length) + ")."); // submit input data LOGGER.info("Submit input data."); RByteIOClient fileClient = pSms.getImport("/" + TARGET_SYSTEM_INPUT_FILE_NAME); fileClient.write(dataCompressed); }
protected void submitInputData(StorageClient pSms, UnicoreAlgorithmInput pAlgorithmInput) throws Exception { RByteIOClient fileClient = pSms.getImport("/" + TARGET_SYSTEM_INPUT_FILE_NAME); // create serialized input data LOGGER.info("Serialize input data."); byte[] data = Compression.toByteArray(pAlgorithmInput); // create input data Properties unicoreProperties = UnicoreAlgorithmRepository.getInstance().getUnicoreProperties(); byte[] dataCompressed = null; if (Boolean.parseBoolean(unicoreProperties.getProperty(UnicoreAlgorithmRepository.CFG_COMPRESSION))) { dataCompressed = Compression.createCompressedData(data, Boolean.parseBoolean(unicoreProperties .getProperty(UnicoreAlgorithmRepository.CFG_COMPRESSION))); LOGGER.info("Compress input data (ratio: " + ((double) dataCompressed.length) / ((double) data.length) + ")."); } // submit input data if (Boolean.parseBoolean(unicoreProperties.getProperty(UnicoreAlgorithmRepository.CFG_COMPRESSION))) { LOGGER.info("Submit compressed input data."); fileClient.write(dataCompressed); } else { LOGGER.info("Submit input data."); fileClient.write(data); } }