/** * @param errorHandlerRID the remoteID of the error handler. * @param nodeID the ID of the (fake) node this Container is instantiated on * @param containedID the ID used to identify this container uniquely * @param folder the folder in which logs etc. will be deposited */ ProcessContainer(final String errorHandlerRID, final String nodeID, final String containedID, final File folder, final int megaBytes, final int numberOfCores, final REEFFileNames fileNames, final ReefRunnableProcessObserver processObserver) { this.errorHandlerRID = errorHandlerRID; this.nodeID = nodeID; this.containedID = containedID; this.folder = folder; this.megaBytes = megaBytes; this.numberOfCores = numberOfCores; this.fileNames = fileNames; this.processObserver = processObserver; this.reefFolder = new File(folder, fileNames.getREEFFolderName()); this.localFolder = new File(reefFolder, fileNames.getLocalFolderName()); this.localFolder.mkdirs(); this.globalFolder = new File(reefFolder, fileNames.getGlobalFolderName()); this.globalFolder.mkdirs(); }
private File makeGlobalJar() throws IOException { final File jarFile = new File( this.fileNames.getGlobalFolderName() + this.fileNames.getJarFileSuffix()); new JARFileMaker(jarFile).addChildren(this.fileNames.getGlobalFolder()).close(); return jarFile; }
private File makejobSubmissionFolder() throws IOException { return Files.createTempDirectory(this.fileNames.getJobFolderPrefix()).toFile(); } }
/** * @return the folder holding the files global to all containers. */ public File getGlobalFolder() { return new File(getREEFFolder(), getGlobalFolderName()); }
"/tmp/" + this.filenames.getJobFolderPrefix() + applicationId.getId() + "/"); resources.put(this.filenames.getREEFFolderName(), this.makeLocalResourceForJarFile(uploadedJobJarPath)); .setErrorHandlerRID(jobSubmissionProto.getRemoteId()) .setLaunchID(jobSubmissionProto.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath()) .setMemory(amMemory) .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName()) .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName()) .build();
public File createJobSubmissionJAR( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final Configuration driverConfiguration) throws IOException { // Copy all files to a local job submission folder final File jobSubmissionFolder = makejobSubmissionFolder(); LOG.log(Level.FINE, "Staging submission in {0}", jobSubmissionFolder); final File localFolder = new File(jobSubmissionFolder, this.fileNames.getLocalFolderName()); final File globalFolder = new File(jobSubmissionFolder, this.fileNames.getGlobalFolderName()); this.copy(jobSubmissionProto.getGlobalFileList(), globalFolder); this.copy(jobSubmissionProto.getLocalFileList(), localFolder); // Store the Driver Configuration in the JAR file. this.configurationSerializer.toFile( driverConfiguration, new File(localFolder, this.fileNames.getDriverConfigurationName())); // Create a JAR File for the submission final File jarFile = File.createTempFile(this.fileNames.getJobFolderPrefix(), this.fileNames.getJarFileSuffix()); LOG.log(Level.FINE, "Creating job submission jar file: {0}", jarFile); new JARFileMaker(jarFile).addChildren(jobSubmissionFolder).close(); if (this.deleteTempFilesOnExit) { LOG.log(Level.FINE, "Deleting the temporary job folder [{0}] and marking the jar file [{1}] for deletion after the JVM exits.", new Object[]{jobSubmissionFolder.getAbsolutePath(), jarFile.getAbsolutePath()}); jobSubmissionFolder.delete(); jarFile.deleteOnExit(); } else { LOG.log(Level.FINE, "Keeping the temporary job folder [{0}] and jar file [{1}] available after job submission.", new Object[]{jobSubmissionFolder.getAbsolutePath(), jarFile.getAbsolutePath()}); } return jarFile; }
/** * Assembles the command to execute the Driver in list form. */ private List<String> getCommandList( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { return new JavaLaunchCommandBuilder() .setJavaPath("%JAVA_HOME%/bin/java") .setErrorHandlerRID(jobSubmissionProto.getRemoteId()) .setLaunchID(jobSubmissionProto.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath()) .setMemory(jobSubmissionProto.getDriverMemory()) .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName()) .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName()) .build(); }
result.putAll(getGlobalResources()); final File localStagingFolder = this.tempFileCreator.createTempDirectory(this.fileNames.getEvaluatorFolderPrefix()); localStagingFolder, this.fileNames.getEvaluatorConfigurationName()); this.configurationSerializer.toFile( makeEvaluatorConfiguration(resourceLaunchProto), configurationFile); this.fileNames.getEvaluatorFolderPrefix(), this.fileNames.getJarFileSuffix()); new JARFileMaker(localFile).addChildren(localStagingFolder).close(); result.put(this.fileNames.getLocalFolderPath(), makeLocalResourceForJarFile(pathToEvaluatorJar));
/** * @return the folder und which all REEF files are stored. */ public File getREEFFolder() { return new File(getREEFFolderName()); }
@Inject ClasspathProvider(final RuntimeClasspathProvider runtimeClasspathProvider, final REEFFileNames reefFileNames) { final List<String> baseClasspath = Arrays.asList( reefFileNames.getLocalFolderPath() + "/*", reefFileNames.getGlobalFolderPath() + "/*"); // Assemble the driver classpath final List<String> runtimeDriverClasspathPrefix = runtimeClasspathProvider.getDriverClasspathPrefix(); final List<String> runtimeDriverClasspathSuffix = runtimeClasspathProvider.getDriverClasspathSuffix(); final List<String> driverClasspath = new ArrayList<>(baseClasspath.size() + runtimeDriverClasspathPrefix.size() + runtimeDriverClasspathSuffix.size()); driverClasspath.addAll(runtimeDriverClasspathPrefix); driverClasspath.addAll(baseClasspath); driverClasspath.addAll(runtimeDriverClasspathSuffix); this.driverClasspath = Collections.unmodifiableList(driverClasspath); // Assemble the evaluator classpath final List<String> runtimeEvaluatorClasspathPrefix = runtimeClasspathProvider.getEvaluatorClasspathPrefix(); final List<String> runtimeEvaluatorClasspathSuffix = runtimeClasspathProvider.getEvaluatorClasspathSuffix(); final List<String> evaluatorClasspath = new ArrayList<>(runtimeEvaluatorClasspathPrefix.size() + baseClasspath.size() + runtimeEvaluatorClasspathSuffix.size()); evaluatorClasspath.addAll(runtimeEvaluatorClasspathPrefix); evaluatorClasspath.addAll(baseClasspath); evaluatorClasspath.addAll(runtimeEvaluatorClasspathSuffix); this.evaluatorClasspath = Collections.unmodifiableList(evaluatorClasspath); }
.setErrorHandlerRID(resourceLaunchProto.getRemoteId()) .setLaunchID(resourceLaunchProto.getIdentifier()) .setConfigurationFileName(this.filenames.getEvaluatorConfigurationPath()) .setMemory((int) (this.jvmHeapFactor * container.getResource().getMemory())) .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getEvaluatorStderrFileName()) .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getEvaluatorStdoutFileName()) .build();
c.addGlobalFiles(this.fileNames.getGlobalFolder()); c.addLocalFiles(getLocalFiles(launchRequest)); final File evaluatorConfigurationFile = new File(c.getFolder(), fileNames.getEvaluatorConfigurationPath()); .setErrorHandlerRID(this.remoteManager.getMyIdentifier()) .setLaunchID(c.getNodeID()) .setConfigurationFileName(this.fileNames.getEvaluatorConfigurationPath()) .setMemory((int) (this.jvmHeapFactor * c.getMemory())) .build();
final File runtimeConfigurationFile = new File(driverFolder, this.filenames.getDriverConfigurationPath()); this.configurationSerializer.toFile(driverConfiguration, runtimeConfigurationFile); .setErrorHandlerRID(t.getRemoteId()) .setLaunchID(t.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath()) .setMemory(DRIVER_MEMORY)
private Map<String, LocalResource> setup() throws IOException { final Map<String, LocalResource> result = new HashMap<>(1); final Path pathToGlobalJar = this.uploadToJobFolder(makeGlobalJar()); result.put(this.fileNames.getGlobalFolderPath(), makeLocalResourceForJarFile(pathToGlobalJar)); return result; }
.setResource(getResource(jobSubmissionProto)) .setContainerInfo(new ContainerInfo() .addFileResource(this.filenames.getREEFFolderName(), uploadedFile) .addCommand(command));
/** * Copies this set of files to the destination folder given. * * Will attempt to create symbolic links for the files to the destination * folder. If the filesystem does not support symbolic links or the user * does not have appropriate permissions, the entire file will be copied instead. * * @param destinationFolder the folder the files shall be copied to. * @throws IOException if one or more of the copies fail. */ public void copyTo(final File destinationFolder) throws IOException { destinationFolder.mkdirs(); final File reefFolder = new File(destinationFolder, fileNames.getREEFFolderName()); final File localFolder = new File(reefFolder, fileNames.getLocalFolderName()); final File globalFolder = new File(reefFolder, fileNames.getGlobalFolderName()); localFolder.mkdirs(); globalFolder.mkdirs(); try { this.localFiles.createSymbolicLinkTo(localFolder); this.localLibs.createSymbolicLinkTo(localFolder); this.globalLibs.createSymbolicLinkTo(globalFolder); this.globalFiles.createSymbolicLinkTo(globalFolder); } catch (IOException e) { this.localFiles.copyTo(localFolder); this.localLibs.copyTo(localFolder); this.globalLibs.copyTo(globalFolder); this.globalFiles.copyTo(globalFolder); } }