private static TestShuffleDataContext createSomeData() throws IOException { Random rand = new Random(123); TestShuffleDataContext dataContext = new TestShuffleDataContext(10, 5); dataContext.create(); dataContext.insertSortShuffleData(rand.nextInt(1000), rand.nextInt(1000), new byte[][] { "ABC".getBytes(StandardCharsets.UTF_8), "DEF".getBytes(StandardCharsets.UTF_8)}); return dataContext; } }
@AfterClass public static void afterAll() { dataContext.cleanup(); }
private static void createNonShuffleFiles(TestShuffleDataContext dataContext) throws IOException { // Create spill file(s) dataContext.insertSpillData(); } }
private static TestShuffleDataContext createDataContext() { TestShuffleDataContext dataContext = new TestShuffleDataContext(10, 5); dataContext.create(); return dataContext; }
@Test (expected = RuntimeException.class) public void testRegisterInvalidExecutor() throws Exception { registerExecutor("exec-1", dataContext0.createExecutorInfo("unknown sort manager")); }
@Test public void cleanupUsesExecutor() throws IOException { TestShuffleDataContext dataContext = createSomeData(); AtomicBoolean cleanupCalled = new AtomicBoolean(false); // Executor which does nothing to ensure we're actually using it. Executor noThreadExecutor = runnable -> cleanupCalled.set(true); ExternalShuffleBlockResolver manager = new ExternalShuffleBlockResolver(conf, null, noThreadExecutor); manager.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); manager.applicationRemoved("app", true); assertTrue(cleanupCalled.get()); assertStillThere(dataContext); dataContext.cleanup(); assertCleanedUp(dataContext); }
private static void createShuffleFiles(TestShuffleDataContext dataContext) throws IOException { Random rand = new Random(123); dataContext.insertSortShuffleData(rand.nextInt(1000), rand.nextInt(1000), new byte[][] { "ABC".getBytes(StandardCharsets.UTF_8), "DEF".getBytes(StandardCharsets.UTF_8)}); }
@Test (expected = RuntimeException.class) public void testRegisterInvalidExecutor() throws Exception { registerExecutor("exec-1", dataContext0.createExecutorInfo("unknown sort manager")); }
@Test public void cleanupUsesExecutor() throws IOException { TestShuffleDataContext dataContext = createSomeData(); AtomicBoolean cleanupCalled = new AtomicBoolean(false); // Executor which does nothing to ensure we're actually using it. Executor noThreadExecutor = runnable -> cleanupCalled.set(true); ExternalShuffleBlockResolver manager = new ExternalShuffleBlockResolver(conf, null, noThreadExecutor); manager.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); manager.applicationRemoved("app", true); assertTrue(cleanupCalled.get()); assertStillThere(dataContext); dataContext.cleanup(); assertCleanedUp(dataContext); }
private static TestShuffleDataContext createDataContext() { TestShuffleDataContext dataContext = new TestShuffleDataContext(10, 5); dataContext.create(); return dataContext; }
private static void createShuffleFiles(TestShuffleDataContext dataContext) throws IOException { Random rand = new Random(123); dataContext.insertSortShuffleData(rand.nextInt(1000), rand.nextInt(1000), new byte[][] { "ABC".getBytes(StandardCharsets.UTF_8), "DEF".getBytes(StandardCharsets.UTF_8)}); }
private static TestShuffleDataContext createSomeData() throws IOException { Random rand = new Random(123); TestShuffleDataContext dataContext = new TestShuffleDataContext(10, 5); dataContext.create(); dataContext.insertSortShuffleData(rand.nextInt(1000), rand.nextInt(1000), new byte[][] { "ABC".getBytes(StandardCharsets.UTF_8), "DEF".getBytes(StandardCharsets.UTF_8)}); return dataContext; } }
@Test (expected = RuntimeException.class) public void testRegisterInvalidExecutor() throws Exception { registerExecutor("exec-1", dataContext0.createExecutorInfo("unknown sort manager")); }
@Test public void cleanupUsesExecutor() throws IOException { TestShuffleDataContext dataContext = createSomeData(); AtomicBoolean cleanupCalled = new AtomicBoolean(false); // Executor which does nothing to ensure we're actually using it. Executor noThreadExecutor = runnable -> cleanupCalled.set(true); ExternalShuffleBlockResolver manager = new ExternalShuffleBlockResolver(conf, null, noThreadExecutor); manager.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); manager.applicationRemoved("app", true); assertTrue(cleanupCalled.get()); assertStillThere(dataContext); dataContext.cleanup(); assertCleanedUp(dataContext); }
@AfterClass public static void afterAll() { dataContext.cleanup(); }
private static void createNonShuffleFiles(TestShuffleDataContext dataContext) throws IOException { // Create spill file(s) dataContext.insertSpillData(); } }
private static TestShuffleDataContext createSomeData() throws IOException { Random rand = new Random(123); TestShuffleDataContext dataContext = new TestShuffleDataContext(10, 5); dataContext.create(); dataContext.insertSortShuffleData(rand.nextInt(1000), rand.nextInt(1000), new byte[][] { "ABC".getBytes(StandardCharsets.UTF_8), "DEF".getBytes(StandardCharsets.UTF_8)}); return dataContext; } }
private void cleanupUsesExecutor(boolean withShuffleFiles) throws IOException { TestShuffleDataContext dataContext = initDataContext(withShuffleFiles); AtomicBoolean cleanupCalled = new AtomicBoolean(false); // Executor which does nothing to ensure we're actually using it. Executor noThreadExecutor = runnable -> cleanupCalled.set(true); ExternalShuffleBlockResolver manager = new ExternalShuffleBlockResolver(conf, null, noThreadExecutor); manager.registerExecutor("app", "exec0", dataContext.createExecutorInfo(SORT_MANAGER)); manager.executorRemoved("exec0", "app"); assertTrue(cleanupCalled.get()); assertStillThere(dataContext); }
@AfterClass public static void afterAll() { dataContext.cleanup(); }
@BeforeClass public static void beforeAll() throws IOException { dataContext = new TestShuffleDataContext(2, 5); dataContext.create(); // Write some sort data. dataContext.insertSortShuffleData(0, 0, new byte[][] { sortBlock0.getBytes(StandardCharsets.UTF_8), sortBlock1.getBytes(StandardCharsets.UTF_8)}); }