FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[0]), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }, blobService.getPermanentBlobService()), new TestingTaskManagerRuntimeInfo(),
@Override public Path call() throws IOException { // let exceptions propagate. we can retrieve them later from // the future and report them upon access to the result copy(filePath, cachedPath, this.executable); return cachedPath; } }
/** * Deletes the local file after a 5 second delay. * * @param name The name under which the file is registered. * @param jobID The ID of the job for which the file is copied. */ public void deleteTmpFile(String name, JobID jobID) { DeleteProcess dp = new DeleteProcess(lock, entries, name, jobID); executorService.schedule(dp, 5000L, TimeUnit.MILLISECONDS); }
cp = new CopyFromBlobProcess(entry, jobID, blobService, new Path(tempDirToUse.getAbsolutePath())); } else { cp = new CopyFromDFSProcess(entry, new Path(tempDirToUse.getAbsolutePath()));
@VisibleForTesting FileCache(String[] tempDirectories, PermanentBlobService blobService, ScheduledExecutorService executorService, long cleanupInterval) throws IOException { Preconditions.checkNotNull(tempDirectories); this.cleanupInterval = cleanupInterval; storageDirectories = new File[tempDirectories.length]; for (int i = 0; i < tempDirectories.length; i++) { String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString(); storageDirectories[i] = new File(tempDirectories[i], cacheDirName); String path = storageDirectories[i].getAbsolutePath(); if (storageDirectories[i].mkdirs()) { LOG.info("User file cache uses directory " + path); } else { LOG.error("User file cache cannot create directory " + path); // delete all other directories we created so far for (int k = 0; k < i; k++) { if (!storageDirectories[k].delete()) { LOG.warn("User file cache cannot remove prior directory " + storageDirectories[k].getAbsolutePath()); } } throw new IOException("File cache cannot create temp storage directory: " + path); } } this.shutdownHook = createShutdownHook(this, LOG); this.entries = new HashMap<>(); this.jobRefHolders = new HashMap<>(); this.executorService = executorService; this.blobService = blobService; }
@Override public void run() { try { cache.shutdown(); } catch (Throwable t) { logger.error("Error during shutdown of file cache via JVM shutdown hook: " + t.getMessage(), t); } } });
private void removeCachedFiles(Map<String, Future<Path>> entries, FileCache fileCache) { // cancel and release all distributed cache files try { for (Map.Entry<String, Future<Path>> entry : entries.entrySet()) { String name = entry.getKey(); try { fileCache.deleteTmpFile(name, jobId); } catch (Exception e) { // unpleasant, but we continue LOG.error("Distributed Cache could not remove cached file registered under '" + name + "'.", e); } } } catch (Throwable t) { LOG.error("Error while removing cached local files from distributed cache."); } }
FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[0]), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }, blobService.getPermanentBlobService()), new TestingTaskManagerRuntimeInfo(),
cp = new CopyFromBlobProcess(entry, jobID, blobService, new Path(tempDirToUse.getAbsolutePath())); } else { cp = new CopyFromDFSProcess(entry, new Path(tempDirToUse.getAbsolutePath()));
public void releaseJob(JobID jobId, ExecutionAttemptID executionId) { checkNotNull(jobId); synchronized (lock) { Set<ExecutionAttemptID> jobRefCounter = jobRefHolders.get(jobId); if (jobRefCounter == null || jobRefCounter.isEmpty()) { return; } jobRefCounter.remove(executionId); if (jobRefCounter.isEmpty()) { executorService.schedule(new DeleteProcess(jobId), cleanupInterval, TimeUnit.MILLISECONDS); } } }
@VisibleForTesting FileCache(String[] tempDirectories, PermanentBlobService blobService, ScheduledExecutorService executorService, long cleanupInterval) throws IOException { Preconditions.checkNotNull(tempDirectories); this.cleanupInterval = cleanupInterval; storageDirectories = new File[tempDirectories.length]; for (int i = 0; i < tempDirectories.length; i++) { String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString(); storageDirectories[i] = new File(tempDirectories[i], cacheDirName); String path = storageDirectories[i].getAbsolutePath(); if (storageDirectories[i].mkdirs()) { LOG.info("User file cache uses directory " + path); } else { LOG.error("User file cache cannot create directory " + path); // delete all other directories we created so far for (int k = 0; k < i; k++) { if (!storageDirectories[k].delete()) { LOG.warn("User file cache cannot remove prior directory " + storageDirectories[k].getAbsolutePath()); } } throw new IOException("File cache cannot create temp storage directory: " + path); } } this.shutdownHook = createShutdownHook(this, LOG); this.entries = new HashMap<>(); this.jobRefHolders = new HashMap<>(); this.executorService = executorService; this.blobService = blobService; }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task. // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { LOG.error("could not copy file to local file cache.", ioe); } } } }
@Override public void start() throws Exception { super.start(); // start by connecting to the ResourceManager try { resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); } catch (Exception e) { onFatalError(e); } // tell the task slot table who's responsible for the task slot actions taskSlotTable.start(new SlotActionsImpl()); // start the job leader service jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl()); fileCache = new FileCache(taskManagerConfiguration.getTmpDirectories(), blobCacheService.getPermanentBlobService()); startRegistrationTimeout(); }
cp = new CopyFromBlobProcess(entry, jobID, blobService, new Path(tempDirToUse.getAbsolutePath())); } else { cp = new CopyFromDFSProcess(entry, new Path(tempDirToUse.getAbsolutePath()));
public void releaseJob(JobID jobId, ExecutionAttemptID executionId) { checkNotNull(jobId); synchronized (lock) { Set<ExecutionAttemptID> jobRefCounter = jobRefHolders.get(jobId); if (jobRefCounter == null || jobRefCounter.isEmpty()) { return; } jobRefCounter.remove(executionId); if (jobRefCounter.isEmpty()) { executorService.schedule(new DeleteProcess(jobId), cleanupInterval, TimeUnit.MILLISECONDS); } } }
@VisibleForTesting FileCache(String[] tempDirectories, PermanentBlobService blobService, ScheduledExecutorService executorService, long cleanupInterval) throws IOException { Preconditions.checkNotNull(tempDirectories); this.cleanupInterval = cleanupInterval; storageDirectories = new File[tempDirectories.length]; for (int i = 0; i < tempDirectories.length; i++) { String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString(); storageDirectories[i] = new File(tempDirectories[i], cacheDirName); String path = storageDirectories[i].getAbsolutePath(); if (storageDirectories[i].mkdirs()) { LOG.info("User file cache uses directory " + path); } else { LOG.error("User file cache cannot create directory " + path); // delete all other directories we created so far for (int k = 0; k < i; k++) { if (!storageDirectories[k].delete()) { LOG.warn("User file cache cannot remove prior directory " + storageDirectories[k].getAbsolutePath()); } } throw new IOException("File cache cannot create temp storage directory: " + path); } } this.shutdownHook = createShutdownHook(this, LOG); this.entries = new HashMap<>(); this.jobRefHolders = new HashMap<>(); this.executorService = executorService; this.blobService = blobService; }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task. // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { LOG.error("could not copy file to local file cache.", ioe); } } } }
@Override public void start() throws Exception { super.start(); // start by connecting to the ResourceManager try { resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); } catch (Exception e) { onFatalError(e); } // tell the task slot table who's responsible for the task slot actions taskSlotTable.start(new SlotActionsImpl()); // start the job leader service jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl()); fileCache = new FileCache(taskManagerConfiguration.getTmpDirectories(), blobCacheService.getPermanentBlobService()); startRegistrationTimeout(); }
public void releaseJob(JobID jobId, ExecutionAttemptID executionId) { checkNotNull(jobId); synchronized (lock) { Set<ExecutionAttemptID> jobRefCounter = jobRefHolders.get(jobId); if (jobRefCounter == null || jobRefCounter.isEmpty()) { LOG.warn("improper use of releaseJob() without a matching number of createTmpFiles() calls for jobId " + jobId); return; } jobRefCounter.remove(executionId); if (jobRefCounter.isEmpty()) { executorService.schedule(new DeleteProcess(jobId), cleanupInterval, TimeUnit.MILLISECONDS); } } }
@Override public void start() throws Exception { super.start(); // start by connecting to the ResourceManager try { startRegistrationTimeout(); resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); } catch (Exception e) { onFatalError(e); } // tell the task slot table who's responsible for the task slot actions taskSlotTable.start(new SlotActionsImpl()); // start the job leader service jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl()); fileCache = new FileCache(taskManagerConfiguration.getTmpDirectories(), blobCacheService.getPermanentBlobService()); }