FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[0]), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }, blobService.getPermanentBlobService()), new TestingTaskManagerRuntimeInfo(),
@Override public Path call() throws IOException { // let exceptions propagate. we can retrieve them later from // the future and report them upon access to the result copy(filePath, cachedPath, this.executable); return cachedPath; } }
DistributedCache.readFileInfoFromConfig(jobConfiguration)) { LOG.info("Obtaining local cache file for '{}'.", entry.getKey()); Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId, executionId); distributedCacheEntries.put(entry.getKey(), cp); fileCache.releaseJob(jobId, executionId); blobService.getPermanentBlobService().releaseJob(jobId);
@Override public void run() { try { cache.shutdown(); } catch (Throwable t) { logger.error("Error during shutdown of file cache via JVM shutdown hook: " + t.getMessage(), t); } } });
public FileCache(String[] tempDirectories) throws IOException { Preconditions.checkNotNull(tempDirectories); storageDirectories = new File[tempDirectories.length]; for (int i = 0; i < tempDirectories.length; i++) { String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString(); storageDirectories[i] = new File(tempDirectories[i], cacheDirName); String path = storageDirectories[i].getAbsolutePath(); if (storageDirectories[i].mkdirs()) { LOG.info("User file cache uses directory " + path); } else { LOG.error("User file cache cannot create directory " + path); // delete all other directories we created so far for (int k = 0; k < i; k++) { if (!storageDirectories[k].delete()) { LOG.warn("User file cache cannot remove prior directory " + storageDirectories[k].getAbsolutePath()); } } throw new IOException("File cache cannot create temp storage directory: " + path); } } this.shutdownHook = createShutdownHook(this, LOG); this.entries = new HashMap<JobID, Map<String, Tuple4<Integer, File, Path, Future<Path>>>>(); this.executorService = Executors.newScheduledThreadPool(10, new ExecutorThreadFactory("flink-file-cache")); }
Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId); distributedCacheEntries.put(entry.getKey(), cp);
DistributedCache.readFileInfoFromConfig(jobConfiguration)) { LOG.info("Obtaining local cache file for '{}'.", entry.getKey()); Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId, executionId); distributedCacheEntries.put(entry.getKey(), cp); fileCache.releaseJob(jobId, executionId); blobService.getPermanentBlobService().releaseJob(jobId);
fileCache.shutdown(); } catch (Throwable t) { throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
@VisibleForTesting FileCache(String[] tempDirectories, PermanentBlobService blobService, ScheduledExecutorService executorService, long cleanupInterval) throws IOException { Preconditions.checkNotNull(tempDirectories); this.cleanupInterval = cleanupInterval; storageDirectories = new File[tempDirectories.length]; for (int i = 0; i < tempDirectories.length; i++) { String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString(); storageDirectories[i] = new File(tempDirectories[i], cacheDirName); String path = storageDirectories[i].getAbsolutePath(); if (storageDirectories[i].mkdirs()) { LOG.info("User file cache uses directory " + path); } else { LOG.error("User file cache cannot create directory " + path); // delete all other directories we created so far for (int k = 0; k < i; k++) { if (!storageDirectories[k].delete()) { LOG.warn("User file cache cannot remove prior directory " + storageDirectories[k].getAbsolutePath()); } } throw new IOException("File cache cannot create temp storage directory: " + path); } } this.shutdownHook = createShutdownHook(this, LOG); this.entries = new HashMap<>(); this.jobRefHolders = new HashMap<>(); this.executorService = executorService; this.blobService = blobService; }
FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[0]), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }, blobService.getPermanentBlobService()), new TestingTaskManagerRuntimeInfo(),
DistributedCache.readFileInfoFromConfig(jobConfiguration)) { LOG.info("Obtaining local cache file for '{}'.", entry.getKey()); Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId, executionId); distributedCacheEntries.put(entry.getKey(), cp); fileCache.releaseJob(jobId, executionId); blobService.getPermanentBlobService().releaseJob(jobId);
fileCache.shutdown(); } catch (Throwable t) { throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
@VisibleForTesting FileCache(String[] tempDirectories, PermanentBlobService blobService, ScheduledExecutorService executorService, long cleanupInterval) throws IOException { Preconditions.checkNotNull(tempDirectories); this.cleanupInterval = cleanupInterval; storageDirectories = new File[tempDirectories.length]; for (int i = 0; i < tempDirectories.length; i++) { String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString(); storageDirectories[i] = new File(tempDirectories[i], cacheDirName); String path = storageDirectories[i].getAbsolutePath(); if (storageDirectories[i].mkdirs()) { LOG.info("User file cache uses directory " + path); } else { LOG.error("User file cache cannot create directory " + path); // delete all other directories we created so far for (int k = 0; k < i; k++) { if (!storageDirectories[k].delete()) { LOG.warn("User file cache cannot remove prior directory " + storageDirectories[k].getAbsolutePath()); } } throw new IOException("File cache cannot create temp storage directory: " + path); } } this.shutdownHook = createShutdownHook(this, LOG); this.entries = new HashMap<>(); this.jobRefHolders = new HashMap<>(); this.executorService = executorService; this.blobService = blobService; }
FileCache.copy(new Path(entry.getValue().toURI().toString()), targetFilePath, false);
@Override public void start() throws Exception { super.start(); // start by connecting to the ResourceManager try { resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); } catch (Exception e) { onFatalError(e); } // tell the task slot table who's responsible for the task slot actions taskSlotTable.start(new SlotActionsImpl()); // start the job leader service jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl()); fileCache = new FileCache(taskManagerConfiguration.getTmpDirectories(), blobCacheService.getPermanentBlobService()); startRegistrationTimeout(); }
/** * Called to shut down the TaskManager. The method closes all TaskManager services. */ @Override public void shutDown() throws Exception { log.info("Stopping TaskManager {}.", getAddress()); Exception exception = null; taskSlotTable.stop(); if (isConnectedToResourceManager()) { resourceManagerConnection.close(); } jobManagerHeartbeatManager.stop(); resourceManagerHeartbeatManager.stop(); ioManager.shutdown(); memoryManager.shutdown(); networkEnvironment.shutdown(); fileCache.shutdown(); try { super.shutDown(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } if (exception != null) { ExceptionUtils.rethrowException(exception, "Error while shutting the TaskExecutor down."); } log.info("Stopped TaskManager {}.", getAddress()); }
@VisibleForTesting FileCache(String[] tempDirectories, PermanentBlobService blobService, ScheduledExecutorService executorService, long cleanupInterval) throws IOException { Preconditions.checkNotNull(tempDirectories); this.cleanupInterval = cleanupInterval; storageDirectories = new File[tempDirectories.length]; for (int i = 0; i < tempDirectories.length; i++) { String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString(); storageDirectories[i] = new File(tempDirectories[i], cacheDirName); String path = storageDirectories[i].getAbsolutePath(); if (storageDirectories[i].mkdirs()) { LOG.info("User file cache uses directory " + path); } else { LOG.error("User file cache cannot create directory " + path); // delete all other directories we created so far for (int k = 0; k < i; k++) { if (!storageDirectories[k].delete()) { LOG.warn("User file cache cannot remove prior directory " + storageDirectories[k].getAbsolutePath()); } } throw new IOException("File cache cannot create temp storage directory: " + path); } } this.shutdownHook = createShutdownHook(this, LOG); this.entries = new HashMap<>(); this.jobRefHolders = new HashMap<>(); this.executorService = executorService; this.blobService = blobService; }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task. // we unwrap the file system to get raw streams without safety net FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri()); FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri()); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { LOG.error("could not copy file to local file cache.", ioe); } } } }
@Override public void start() throws Exception { super.start(); // start by connecting to the ResourceManager try { resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); } catch (Exception e) { onFatalError(e); } // tell the task slot table who's responsible for the task slot actions taskSlotTable.start(new SlotActionsImpl()); // start the job leader service jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl()); fileCache = new FileCache(taskManagerConfiguration.getTmpDirectories(), blobCacheService.getPermanentBlobService()); startRegistrationTimeout(); }
fileCache.shutdown(); } catch (Throwable t) { throwable = ExceptionUtils.firstOrSuppressed(t, throwable);