@Override public void onComplete(String streamId) throws IOException { listener.onBlockFetchSuccess(blockIds[chunkIndex], channel.closeAndRead()); if (!downloadFileManager.registerTempFileToClean(targetFile)) { targetFile.delete(); } }
DownloadCallback(int chunkIndex) throws IOException { this.targetFile = downloadFileManager.createTempFile(transportConf); this.channel = targetFile.openForWriting(); this.chunkIndex = chunkIndex; }
@Override public void onSuccess(int chunkIndex, ManagedBuffer buffer) { // On receipt of a chunk, pass it upwards as a block. listener.onBlockFetchSuccess(blockIds[chunkIndex], buffer); }
@Test public void noCleanupAndCleanup() throws IOException { TestShuffleDataContext dataContext = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", false /* cleanup */); assertStillThere(dataContext); resolver.registerExecutor("app", "exec1", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true /* cleanup */); assertCleanedUp(dataContext); }
private void cleanupOnRemovedExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
private static TestShuffleDataContext initDataContextWithoutShuffleFiles() throws IOException { TestShuffleDataContext dataContext = createDataContext(); createNonShuffleFiles(dataContext); return dataContext; }
private static TestShuffleDataContext initDataContext(boolean withShuffleFiles) throws IOException { if (withShuffleFiles) { return initDataContextWithShuffleFiles(); } else { return initDataContextWithoutShuffleFiles(); } }
@Test public void noCleanupAndCleanup() throws IOException { TestShuffleDataContext dataContext = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", false /* cleanup */); assertStillThere(dataContext); resolver.registerExecutor("app", "exec1", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true /* cleanup */); assertCleanedUp(dataContext); }
private void cleanupOnlyRegisteredExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec1", "app"); assertStillThere(dataContext); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
@Override public void onComplete(String streamId) throws IOException { listener.onBlockFetchSuccess(blockIds[chunkIndex], channel.closeAndRead()); if (!downloadFileManager.registerTempFileToClean(targetFile)) { targetFile.delete(); } }
@Override public void onSuccess(int chunkIndex, ManagedBuffer buffer) { // On receipt of a chunk, pass it upwards as a block. listener.onBlockFetchSuccess(blockIds[chunkIndex], buffer); }
DownloadCallback(int chunkIndex) throws IOException { this.targetFile = downloadFileManager.createTempFile(transportConf); this.channel = targetFile.openForWriting(); this.chunkIndex = chunkIndex; }
private static TestShuffleDataContext initDataContextWithoutShuffleFiles() throws IOException { TestShuffleDataContext dataContext = createDataContext(); createNonShuffleFiles(dataContext); return dataContext; }
@Test public void cleanupMultipleExecutors() throws IOException { TestShuffleDataContext dataContext0 = createSomeData(); TestShuffleDataContext dataContext1 = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext0.createExecutorInfo(SORT_MANAGER)); resolver.registerExecutor("app", "exec1", dataContext1.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true); assertCleanedUp(dataContext0); assertCleanedUp(dataContext1); }
private void cleanupOnRemovedExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
@Test public void noCleanupAndCleanup() throws IOException { TestShuffleDataContext dataContext = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", false /* cleanup */); assertStillThere(dataContext); resolver.registerExecutor("app", "exec1", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true /* cleanup */); assertCleanedUp(dataContext); }
private void cleanupOnlyRegisteredExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); resolver.executorRemoved("exec1", "app"); assertStillThere(dataContext); resolver.executorRemoved("exec0", "app"); assertCleanedUp(dataContext); }
@Test public void cleanupMultipleExecutors() throws IOException { TestShuffleDataContext dataContext0 = createSomeData(); TestShuffleDataContext dataContext1 = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext0.createExecutorInfo(SORT_MANAGER)); resolver.registerExecutor("app", "exec1", dataContext1.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true); assertCleanedUp(dataContext0); assertCleanedUp(dataContext1); }
private void cleanupUsesExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); AtomicBoolean cleanupCalled = new AtomicBoolean(false); // Executor which does nothing to ensure we're actually using it. Executor noThreadExecutor = runnable -> cleanupCalled.set(true); ExternalShuffleBlockResolver manager = new ExternalShuffleBlockResolver(conf, null, noThreadExecutor); manager.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); manager.executorRemoved("exec0", "app"); assertTrue(cleanupCalled.get()); assertStillThere(dataContext); }
@Test public void cleanupMultipleExecutors() throws IOException { TestShuffleDataContext dataContext0 = createSomeData(); TestShuffleDataContext dataContext1 = createSomeData(); ExternalShuffleBlockResolver resolver = new ExternalShuffleBlockResolver(conf, null, sameThreadExecutor); resolver.registerExecutor("app", "exec0", dataContext0.createExecutorInfo(SORT_MANAGER)); resolver.registerExecutor("app", "exec1", dataContext1.createExecutorInfo(SORT_MANAGER)); resolver.applicationRemoved("app", true); assertCleanedUp(dataContext0); assertCleanedUp(dataContext1); }