private void readVersion() throws IOException { Path versionFile = topologyBasicBlobsRootDir.resolve(type.getVersionFileName()); if (!fsOps.fileExists(versionFile)) { version = NOT_DOWNLOADED_VERSION; } else { String ver = FileUtils.readFileToString(versionFile.toFile(), "UTF8").trim(); version = Long.parseLong(ver); } }
/** * Create symlink from the containers directory/artifacts to the artifacts directory. * * @throws IOException on any error */ protected void createArtifactsLink() throws IOException { _type.assertFull(); if (!_symlinksDisabled) { File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId)); File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (_ops.fileExists(workerDir)) { LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId); _ops.createSymlink(new File(workerDir, "artifacts"), topoDir); } } }
/** * @return the user that some operations should be done as. * * @throws IOException on any error */ protected String getWorkerUser() throws IOException { LOG.info("GET worker-user for {}", _workerId); File file = new File(ConfigUtils.workerUserFile(_conf, _workerId)); if (_ops.fileExists(file)) { return _ops.slurpString(file).trim(); } else if (_assignment != null && _assignment.is_set_owner()) { return _assignment.get_owner(); } if (ConfigUtils.isLocalMode(_conf)) { return System.getProperty("user.name"); } else { File f = new File(ConfigUtils.workerArtifactsRoot(_conf)); if (f.exists()) { return Files.getOwner(f.toPath()).getName(); } throw new IllegalStateException("Could not recover the user for " + _workerId); } }
LOG.debug("Removing extraction dest {} in preparation for extraction", extractionDest); fsOps.deleteIfExists(extractionDest.toFile()); if (fsOps.fileExists(extractionTemp)) { fsOps.moveDirectoryPreferAtomic(extractionTemp.toFile(), extractionDest.toFile());
if (!localResourceList.isEmpty()) { File userDir = getLocalUserFileCacheDir(topoOwner); if (!fsOps.fileExists(userDir)) { fsOps.forceMkdir(userDir);
/** * Setup the container to run. By default this creates the needed directories/links in the local file system PREREQUISITE: All needed * blobs and topology, jars/configs have been downloaded and placed in the appropriate locations * * @throws IOException on any error */ protected void setup() throws IOException { _type.assertFull(); if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) { LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment, _supervisorId, _port, _workerId); throw new IllegalStateException("Not all needed files are here!!!!"); } LOG.info("Setting up {}:{}", _supervisorId, _workerId); _ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId))); File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (!_ops.fileExists(workerArtifacts)) { _ops.forceMkdir(workerArtifacts); _ops.setupWorkerArtifactsDir(_assignment.get_owner(), workerArtifacts); } String user = getWorkerUser(); writeLogMetadata(user); saveWorkerUser(user); createArtifactsLink(); createBlobstoreLinks(); }
when(ops.fileExists(workerArtifacts)).thenReturn(true); when(ops.fileExists(workerRoot)).thenReturn(true); when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
when(ops.fileExists(workerArtifacts)).thenReturn(true); when(ops.fileExists(workerRoot)).thenReturn(true); when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump); verify(ops).forceMkdir(new File(workerRoot, "tmp")); verify(ops).forceMkdir(new File(workerRoot, "heartbeats")); verify(ops).fileExists(workerArtifacts);
verify(ops).fileExists(userDir); verify(ops).forceMkdir(userDir);
/** * Create symlink from the containers directory/artifacts to the artifacts directory * @throws IOException on any error */ protected void createArtifactsLink() throws IOException { _type.assertFull(); if (!_symlinksDisabled) { File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId)); File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (_ops.fileExists(workerDir)) { LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId); _ops.createSymlink(new File(workerDir, "artifacts"), topoDir); } } }
/** * @return the user that some operations should be done as. * @throws IOException on any error */ protected String getWorkerUser() throws IOException { LOG.info("GET worker-user for {}", _workerId); File file = new File(ConfigUtils.workerUserFile(_conf, _workerId)); if (_ops.fileExists(file)) { return _ops.slurpString(file).trim(); } else if (_assignment != null && _assignment.is_set_owner()) { return _assignment.get_owner(); } if (ConfigUtils.isLocalMode(_conf)) { return System.getProperty("user.name"); } else { File f = new File(ConfigUtils.workerArtifactsRoot(_conf)); if (f.exists()) { return Files.getOwner(f.toPath()).getName(); } throw new IllegalStateException("Could not recover the user for " + _workerId); } }
@Override public Void call() throws Exception { try { if (_fsOps.fileExists(_stormRoot)) { if (!_fsOps.supportsAtomicDirectoryMove()) { LOG.warn("{} may have partially downloaded blobs, recovering", _topologyId);
if (!_fsOps.fileExists(userDir)) { _fsOps.forceMkdir(userDir);
if (!_ops.fileExists(workerArtifacts)) { _ops.forceMkdir(workerArtifacts); _ops.setupWorkerArtifactsDir(_assignment.get_owner(), workerArtifacts);