if (appDesc.getScratchWorkingDirectory() == null) { appDesc.setScratchWorkingDirectory("/tmp"); if (appDesc.getStaticWorkingDirectory() == null) { String date = new Date().toString(); date = date.replaceAll(" ", "_"); date = date.replaceAll(":", "_"); String tmpDir = appDesc.getScratchWorkingDirectory() + File.separator + appDesc.getApplicationName().getStringValue() + "_" + date + "_" + UUID.randomUUID(); appDesc.setStaticWorkingDirectory(tmpDir); if (appDesc.getInputDataDirectory() == null) { appDesc.setInputDataDirectory(appDesc.getStaticWorkingDirectory() + File.separator + "inputData"); if (appDesc.getOutputDataDirectory() == null) { appDesc.setOutputDataDirectory(appDesc.getStaticWorkingDirectory() + File.separator + "outputData"); if (appDesc.getStandardOutput() == null) { appDesc.setStandardOutput(appDesc.getStaticWorkingDirectory() + File.separator + appDesc.getApplicationName().getStringValue() + ".stdout"); if (appDesc.getStandardError() == null) { appDesc.setStandardError(appDesc.getStaticWorkingDirectory() + File.separator + appDesc.getApplicationName().getStringValue() + ".stderr");
public ApplicationDescription(SchemaType type) { this(); this.appDocument.getApplicationDeploymentDescription().changeType(type); }
public void makeDirectory(InvocationContext invocationContext) throws ProviderException { ApplicationDeploymentDescriptionType app = invocationContext.getExecutionDescription().getApp().getType(); log.info("working diectroy = " + app.getStaticWorkingDirectory()); log.info("temp directory = " + app.getScratchWorkingDirectory()); makeFileSystemDir(app.getStaticWorkingDirectory()); makeFileSystemDir(app.getScratchWorkingDirectory()); makeFileSystemDir(app.getInputDataDirectory()); makeFileSystemDir(app.getOutputDataDirectory()); }
Thread t1 = new ReadStreamWriteFile(process.getInputStream(), app.getStandardOutput()); Thread t2 = new ReadStreamWriteFile(process.getErrorStream(), app.getStandardError()); .append(" on the localHost, working directory = ").append(app.getStaticWorkingDirectory()) .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()).append(" With the status ") .append(String.valueOf(returnValue));
public static ApplicationDescriptor createApplicationDescriptor (ApplicationDescription applicationDescription){ ApplicationDescriptor applicationDescriptor = new ApplicationDescriptor(); applicationDescriptor.setName(applicationDescription.getType().getApplicationName().getStringValue()); applicationDescriptor.setExecutablePath(applicationDescription.getType().getExecutableLocation()); applicationDescriptor.setWorkingDir(applicationDescription.getType().getScratchWorkingDirectory()); if (applicationDescription.getType().getInputDataDirectory() != null && !applicationDescription.getType().getInputDataDirectory().equals("") ){ applicationDescriptor.setInputDir(applicationDescription.getType().getInputDataDirectory()); if (applicationDescription.getType().getOutputDataDirectory() != null && !applicationDescription.getType().getOutputDataDirectory().equals("")){ applicationDescriptor.setOutputDir(applicationDescription.getType().getOutputDataDirectory()); if (applicationDescription.getType().getStaticWorkingDirectory() != null && !applicationDescription.getType().getStaticWorkingDirectory().equals("")){ applicationDescriptor.setStaticWorkigDir(applicationDescription.getType().getStaticWorkingDirectory()); if (applicationDescription.getType().getStandardInput() != null && !applicationDescription.getType().getStandardInput().equals("")){ applicationDescriptor.setStdIn(applicationDescription.getType().getStandardInput()); if (applicationDescription.getType().getStandardOutput() != null && !applicationDescription.getType().getStandardOutput().equals("")){ applicationDescriptor.setStdOut(applicationDescription.getType().getStandardOutput()); if (applicationDescription.getType().getStandardError() != null && !applicationDescription.getType().getStandardError().equals("")){ applicationDescriptor.setStdError(applicationDescription.getType().getStandardError()); NameValuePairType[] environmentArray = applicationDescription.getType().getApplicationEnvironmentArray(); HashMap<String, String> environmentVariableMap = new HashMap<String, String>(); if (environmentArray != null && environmentArray.length != 0){
ApplicationDeploymentDescriptionType.ApplicationName name = ApplicationDeploymentDescriptionType.ApplicationName.Factory.newInstance(); name.setStringValue(applicationDescriptor.getName()); applicationDescription.getType().setApplicationName(name); applicationDescription.getType().setExecutableLocation(applicationDescriptor.getExecutablePath()); applicationDescription.getType().setScratchWorkingDirectory(applicationDescriptor.getWorkingDir()); applicationDescription.getType().setInputDataDirectory(applicationDescriptor.getInputDir()); applicationDescription.getType().setOutputDataDirectory(applicationDescriptor.getOutputDir()); applicationDescription.getType().setStandardInput(applicationDescriptor.getStdIn()); applicationDescription.getType().setStandardOutput(applicationDescriptor.getStdOut()); applicationDescription.getType().setStandardError(applicationDescriptor.getStdError()); applicationDescription.getType().setStaticWorkingDirectory(applicationDescriptor.getStaticWorkigDir()); for(String key : environmentVariables.keySet()) { int i = 0; NameValuePairType nameValuePairType = applicationDescription.getType().addNewApplicationEnvironment(); nameValuePairType.setName(key); nameValuePairType.setValue(environmentVariables.get(key)); applicationDescription.getType().setApplicationEnvironmentArray(appEnviVariablesArray); if (applicationDescriptor.getApplicationDescType().equals(ApplicationDescriptorTypes.HPC_APP_DEP_DESC_TYPE)){ ApplicationDescription appDesc = new ApplicationDescription(HpcApplicationDeploymentType.type); appDesc.getType().setApplicationName(name);
cmdList.add(app.getExecutableLocation()); cmdList.addAll(tmp); NameValuePairType[] env = app.getApplicationEnvironmentArray(); if ((app.getApplicationEnvironmentArray() != null) && (app.getApplicationEnvironmentArray().length != 0) && nv.size() > 0) { builder.environment().putAll(nv); builder.environment().put(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory()); builder.environment().put(GFacConstants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory()); builder.directory(new File(app.getStaticWorkingDirectory()));
URI stdoutURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStandardOutput()); URI stderrURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStandardError()); status.setTransferState(TransferState.COMPLETE); detail.setTransferStatus(status); appDesc.setStandardOutput(stdout); appDesc.setStandardError(stderr); jobExecutionContext.getApplicationContext().setApplicationDeploymentDescription(application); ActualParameter actualParameter = (ActualParameter) output.get(paramName); if ("URIArray".equals(actualParameter.getType().getType().toString())) { URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory()); List<String> outputList = ftp.listDir(outputURI, gssCred); String[] valueList = outputList.toArray(new String[outputList.size()]); stringMap.put(paramName, actualParameter); } else if ("URI".equals(actualParameter.getType().getType().toString())) { URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory()); List<String> outputList = ftp.listDir(outputURI, gssCred); if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
public void setupEnvironment(InvocationContext context) throws ProviderException { ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType(); // input parameter ArrayList<String> tmp = new ArrayList<String>(); for (Iterator<String> iterator = context.getInput().getNames(); iterator.hasNext();) { String key = iterator.next(); tmp.add(context.getInput().getStringValue(key)); } List<String> cmdList = new ArrayList<String>(); /* * Builder Command */ cmdList.add(app.getExecutableLocation()); cmdList.addAll(tmp); // create process builder from command command = InputUtils.buildCommand(cmdList); // redirect StdOut and StdErr // TODO: Make 1> and 2> into static constants. // TODO: This only works for the BASH shell. CSH and TCSH will be // different. command += SPACE + "1>" + SPACE + app.getStandardOutput(); command += SPACE + "2>" + SPACE + app.getStandardError(); }
public Map<String, ?> processOutput(InvocationContext context) throws ProviderException { ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType(); try { // Get the Stdouts and StdErrs String timeStampedServiceName = GfacUtils.createUniqueNameForService(context.getServiceName()); File localStdOutFile = File.createTempFile(timeStampedServiceName, "stdout"); File localStdErrFile = File.createTempFile(timeStampedServiceName, "stderr"); SCPFileTransfer fileTransfer = ssh.newSCPFileTransfer(); fileTransfer.download(app.getStandardOutput(), localStdOutFile.getAbsolutePath()); fileTransfer.download(app.getStandardError(), localStdErrFile.getAbsolutePath()); String stdOutStr = GfacUtils.readFileToString(localStdOutFile.getAbsolutePath()); String stdErrStr = GfacUtils.readFileToString(localStdErrFile.getAbsolutePath()); return OutputUtils.fillOutputFromStdout(context.<ActualParameter> getOutput(), stdOutStr); } catch (XmlException e) { throw new ProviderException("Cannot read output:" + e.getMessage(), e); } catch (ConnectionException e) { throw new ProviderException(e.getMessage(), e); } catch (TransportException e) { throw new ProviderException(e.getMessage(), e); } catch (IOException e) { throw new ProviderException(e.getMessage(), e); } }
.append(job.getRSL()).append(" working directory = ").append(app.getStaticWorkingDirectory()) .append(" temp directory = ").append(app.getScratchWorkingDirectory()) .append(" Globus GateKeeper Endpoint = ").append(gateKeeper);
private boolean isInputDataDirectoryLocal(ApplicationDeploymentDescriptionType appDepDesc){ String inputDataDirectoryPath = appDepDesc.getInputDataDirectory(); File inputDataDirectory = new File(inputDataDirectoryPath); if(inputDataDirectory.exists() && FileUtils.listFiles(inputDataDirectory, null, null).size() > 0){ return true; } return false; }
private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException, IOException { ApplicationDeploymentDescriptionType appDepDesc = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType(); HadoopApplicationDeploymentDescriptionType hadoopAppDesc = (HadoopApplicationDeploymentDescriptionType)appDepDesc; if(appDepDesc.isSetInputDataDirectory() && isInputDataDirectoryLocal(appDepDesc)){ Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir); FileSystem hdfs = FileSystem.get(hadoopConf); hdfs.copyFromLocalFile(new Path(appDepDesc.getInputDataDirectory()), new Path(hadoopAppDesc.getHadoopJobConfiguration().getHdfsInputDirectory())); } }
public List<ApplicationDeploymentDescription> searchDeploymentDescription(String serviceName, String hostName, String applicationName) throws RegistryException { Session session = null; List<ApplicationDeploymentDescription> result = new ArrayList<ApplicationDeploymentDescription>(); try { session = getSession(); Node deploymentNode = getDeploymentNode(session); Node serviceNode = deploymentNode.getNode(serviceName); Node hostNode = serviceNode.getNode(hostName); NodeIterator nodes = hostNode.getNodes(); for (; nodes.hasNext();) { Node app = nodes.nextNode(); Property prop = app.getProperty(XML_PROPERTY_NAME); ApplicationDeploymentDescription appDesc = ApplicationDeploymentDescription.fromXML(prop.getString()); if (appDesc.getType().getApplicationName().getStringValue().matches(applicationName)) { result.add(appDesc); } } } catch (PathNotFoundException e) { return result; } catch (Exception e) { throw new DeploymentDescriptionRetrieveException(e); } finally { closeSession(session); } return result; }
public Map<String, ?> processOutput(InvocationContext context) throws ProviderException { ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType(); try { String stdOutStr = GfacUtils.readFileToString(app.getStandardOutput()); // set to context return OutputUtils.fillOutputFromStdout(context.<ActualParameter> getOutput(), stdOutStr); } catch (XmlException e) { throw new ProviderException("Cannot read output:" + e.getMessage(), e); } catch (IOException io) { throw new ProviderException(io.getMessage(), io); } } }
srcFilePath = app.getOutputDataDirectory() + File.separator + srcFilePath;
public void makeDirectory(InvocationContext context) throws ProviderException { ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType(); Session session = null; try { session = getSession(context); StringBuilder commandString = new StringBuilder(); commandString.append("mkdir -p "); commandString.append(app.getScratchWorkingDirectory()); commandString.append(" ; "); commandString.append("mkdir -p "); commandString.append(app.getStaticWorkingDirectory()); commandString.append(" ; "); commandString.append("mkdir -p "); commandString.append(app.getInputDataDirectory()); commandString.append(" ; "); commandString.append("mkdir -p "); commandString.append(app.getOutputDataDirectory()); Command cmd = session.exec(commandString.toString()); cmd.join(COMMAND_EXECUTION_TIMEOUT, TimeUnit.SECONDS); } catch (ConnectionException e) { throw new ProviderException(e.getMessage(), e); } catch (TransportException e) { throw new ProviderException(e.getMessage(), e); } catch (IOException e) { throw new ProviderException(e.getMessage(), e); } finally { closeSession(session); } }
session.exec("cd " + app.getStaticWorkingDirectory()); NameValuePairType[] env = app.getApplicationEnvironmentArray(); nv.put(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory()); nv.put(GFacConstants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
URI stdoutURI = GfacUtils.createGsiftpURI(endpoint, app.getStandardOutput()); URI stderrURI = GfacUtils.createGsiftpURI(endpoint, app.getStandardError());
.append(job.getRSL()).append(" working directory = ").append(app.getStaticWorkingDirectory()) .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()) .append(" Globus GateKeeper cantact = ").append(gateKeeper); invocationContext.getExecutionContext().getNotifier().info(invocationContext, buf.toString());