@Override public CompletableFuture<Map<K, V>> getAllAsync(List<K> keys) { return readFn.getAllAsync(keys); }
@Override public CompletableFuture<V> getAsync(K key) { return readFn.getAsync(key); }
/** * {@inheritDoc} */ @Override public void close() { readFn.close(); }
@Test public void testGetWithoutRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any()); Map<String, String> result = new HashMap<>(); result.put("foo", "bar"); doReturn(CompletableFuture.completedFuture(result)).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); int times = 0; table.init(TestRemoteTable.getMockContext()); verify(readFn, times(1)).init(any()); assertEquals("bar", table.getAsync("foo").get()); verify(readFn, times(1)).getAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(result, table.getAllAsync(Arrays.asList("foo")).get()); verify(readFn, times(1)).getAllAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.retryTimer.getSnapshot().getMax()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertNull(table.writeRetryMetrics); table.close(); verify(readFn, times(1)).close(); }
private void doTestGet(boolean sync, boolean error, boolean retry) throws Exception { String tableId = "testGet-" + sync + error + retry; TableReadFunction<String, String> readFn = mock(TableReadFunction.class); // Sync is backed by async so needs to mock the async method CompletableFuture<String> future; if (error) { future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("Test exception")); if (!retry) { doReturn(future).when(readFn).getAsync(anyString()); } else { final int [] times = new int[] {0}; doAnswer(args -> times[0]++ == 0 ? future : CompletableFuture.completedFuture("bar")) .when(readFn).getAsync(anyString()); } } else { future = CompletableFuture.completedFuture("bar"); doReturn(future).when(readFn).getAsync(anyString()); } if (retry) { doReturn(true).when(readFn).isRetriable(any()); } RemoteTable<String, String> table = getTable(tableId, readFn, null, retry); Assert.assertEquals("bar", sync ? table.get("foo") : table.getAsync("foo").get()); verify(table.readRateLimiter, times(error && retry ? 2 : 1)).throttle(anyString()); }
@Test public void testGetWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(11)).getAsync(any()); assertEquals(10, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedAttemptsGet() throws Exception { String tableId = "testRetryExhaustedAttempts"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(readFn, times(11)).getAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }
@Test public void testGetThrottling() throws Exception { TableRateLimiter readRateLimiter = mock(TableRateLimiter.class); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any()); Map<String, String> result = new HashMap<>(); result.put("foo", "bar"); doReturn(CompletableFuture.completedFuture(result)).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRateLimitedTable table = new AsyncRateLimitedTable("t1", delegate, readRateLimiter, null, schedExec); table.init(TestRemoteTable.getMockContext()); Assert.assertEquals("bar", table.getAsync("foo").get()); verify(readFn, times(1)).getAsync(any()); verify(readRateLimiter, times(1)).throttle(anyString()); verify(readRateLimiter, times(0)).throttle(anyList()); Assert.assertEquals(result, table.getAllAsync(Arrays.asList("")).get()); verify(readFn, times(1)).getAllAsync(any()); verify(readRateLimiter, times(1)).throttle(anyList()); verify(readRateLimiter, times(1)).throttle(anyString()); }
private TableReadFunction<?, ?> getReadFn(JavaTableConfig tableConfig) { TableReadFunction<?, ?> readFn = deserializeObject(tableConfig, RemoteTableDescriptor.READ_FN); if (readFn != null) { readFn.init(this.context); } return readFn; }
@Test public void testFirstTimeSuccessGet() throws Exception { String tableId = "testFirstTimeSuccessGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals("bar", retryIO.getAsync("foo").get()); verify(readFn, times(1)).getAsync(anyString()); Assert.assertEquals(0, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(1, retryIO.retryMetrics.successCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testRetryEngagedGet() throws Exception { String tableId = "testRetryEngagedGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); int [] times = new int[] {0}; Map<String, String> map = new HashMap<>(); map.put("foo1", "bar1"); map.put("foo2", "bar2"); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times[0] > 0) { future.complete(map); } else { times[0]++; future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals(map, retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(readFn, times(2)).getAllAsync(any()); Assert.assertEquals(1, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }
doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any()); Assert.assertEquals(cachingTable.getAsync("foo").get(), "bar"); records.put("foo1", "bar1"); records.put("foo2", "bar2"); doReturn(CompletableFuture.completedFuture(records)).when(readFn).getAllAsync(any()); Assert.assertEquals(cachingTable.getAllAsync(Arrays.asList("foo1", "foo2")).get(), records); doReturn(CompletableFuture.completedFuture(Collections.singletonMap("foo3", "bar3"))).when(readFn).getAllAsync(any()); records = cachingTable.getAllAsync(Arrays.asList("foo1", "foo2", "foo3")).get(); Assert.assertEquals(records.get("foo3"), "bar3"); doReturn(exFuture).when(readFn).getAllAsync(any()); cachingTable.getAllAsync(Arrays.asList("foo1", "foo2", "foo3")).get();
@Override public void init(Context context) { readFn.init(context); if (writeFn != null) { writeFn.init(context); } }
/** * Fetch single table record for a specified {@code key}. This method must be thread-safe. * The default implementation calls getAsync and blocks on the completion afterwards. * @param key key for the table record * @return table record for the specified {@code key} */ default V get(K key) { try { return getAsync(key).get(); } catch (InterruptedException | ExecutionException e) { throw new SamzaException("GET failed for " + key, e); } }
@Test public void testRetryExhaustedTimeGet() throws Exception { String tableId = "testRetryExhaustedTime"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAsync("foo").get(); Assert.fail(); } catch (ExecutionException e) { } // Conservatively: must be at least 3 attempts with 5ms backoff and 100ms maxDelay verify(readFn, atLeast(3)).getAsync(anyString()); Assert.assertTrue(retryIO.retryMetrics.retryCount.getCount() >= 3); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
/** * Fetch the table {@code records} for specified {@code keys}. This method must be thread-safe. * The default implementation calls getAllAsync and blocks on the completion afterwards. * @param keys keys for the table records * @return all records for the specified keys. */ default Map<K, V> getAll(Collection<K> keys) { try { return getAllAsync(keys).get(); } catch (InterruptedException | ExecutionException e) { throw new SamzaException("GET_ALL failed for " + keys, e); } }
@Test public void testGetAllWithOneRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); AtomicInteger times = new AtomicInteger(); Map<String, String> map = new HashMap<>(); map.put("foo1", "bar1"); map.put("foo2", "bar2"); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times.get() > 0) { future.complete(map); } else { times.incrementAndGet(); future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); assertEquals(map, table.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(readFn, times(2)).getAllAsync(any()); assertEquals(1, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Override public boolean isRetriable(Throwable exception) { return readFn.isRetriable(exception); }