@Test public void testFirstTimeSuccessPut() throws Exception { String tableId = "testFirstTimeSuccessPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(writeFn).putAsync(anyString(), anyString()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); retryIO.putAsync("foo", "bar").get(); verify(writeFn, times(1)).putAsync(anyString(), anyString()); Assert.assertEquals(0, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(1, retryIO.retryMetrics.successCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testFirstTimeSuccessGet() throws Exception { String tableId = "testFirstTimeSuccessGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals("bar", retryIO.getAsync("foo").get()); verify(readFn, times(1)).getAsync(anyString()); Assert.assertEquals(0, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(1, retryIO.retryMetrics.successCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testRetryEngagedGet() throws Exception { String tableId = "testRetryEngagedGet"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); int [] times = new int[] {0}; Map<String, String> map = new HashMap<>(); map.put("foo1", "bar1"); map.put("foo2", "bar2"); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times[0] > 0) { future.complete(map); } else { times[0]++; future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); Assert.assertEquals(map, retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(readFn, times(2)).getAllAsync(any()); Assert.assertEquals(1, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedAttemptsGet() throws Exception { String tableId = "testRetryExhaustedAttempts"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(readFn, times(11)).getAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedAttemptsPut() throws Exception { String tableId = "testRetryExhaustedAttemptsPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).deleteAllAsync(any()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.deleteAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(writeFn, times(11)).deleteAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedTimeGet() throws Exception { String tableId = "testRetryExhaustedTime"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); RetriableReadFunction<String, String> retryIO = new RetriableReadFunction<>(policy, readFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.getAsync("foo").get(); Assert.fail(); } catch (ExecutionException e) { } // Conservatively: must be at least 3 attempts with 5ms backoff and 100ms maxDelay verify(readFn, atLeast(3)).getAsync(anyString()); Assert.assertTrue(retryIO.retryMetrics.retryCount.getCount() >= 3); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryEngagedPut() throws Exception { String tableId = "testRetryEngagedPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAllAsync(any()); doReturn(true).when(writeFn).isRetriable(any()); int [] times = new int[] {0}; List<Entry<String, String>> records = new ArrayList<>(); records.add(new Entry<>("foo1", "bar1")); records.add(new Entry<>("foo2", "bar2")); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times[0] > 0) { future.complete(null); } else { times[0]++; future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(writeFn).putAllAsync(any()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); retryIO.putAllAsync(records).get(); verify(writeFn, times(2)).putAllAsync(any()); Assert.assertEquals(1, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedTimePut() throws Exception { String tableId = "testRetryExhaustedTimePut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).deleteAsync(anyString()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.deleteAsync("foo").get(); Assert.fail(); } catch (ExecutionException e) { } // Conservatively: must be at least 3 attempts with 5ms backoff and 100ms maxDelay verify(writeFn, atLeast(3)).deleteAsync(anyString()); Assert.assertTrue(retryIO.retryMetrics.retryCount.getCount() >= 3); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
Assert.assertEquals(1, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0);
assertEquals(++times, table.writeRetryMetrics.successCount.getCount()); assertEquals(0, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.retryTimer.getSnapshot().getMax()); assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount()); assertNull(table.readRetryMetrics);
@Test public void testGetAllWithOneRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); AtomicInteger times = new AtomicInteger(); Map<String, String> map = new HashMap<>(); map.put("foo1", "bar1"); map.put("foo2", "bar2"); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times.get() > 0) { future.complete(map); } else { times.incrementAndGet(); future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); assertEquals(map, table.getAllAsync(Arrays.asList("foo1", "foo2")).get()); verify(readFn, times(2)).getAllAsync(any()); assertEquals(1, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testPutAllWithOneRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); AtomicInteger times = new AtomicInteger(); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times.get() > 0) { future.complete(null); } else { times.incrementAndGet(); future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(writeFn).putAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); table.putAllAsync(Arrays.asList(new Entry(1, 2))).get(); verify(writeFn, times(2)).putAllAsync(any()); assertEquals(1, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount()); assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testPutWithRetryDisabled() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(false).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).putAsync(any(), any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAsync("foo", "bar").get(); fail(); } catch (ExecutionException e) { } verify(writeFn, times(1)).putAsync(any(), any()); assertEquals(0, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount()); assertEquals(0, table.writeRetryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testGetWithoutRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableReadFunction readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(readFn).getAsync(any()); Map<String, String> result = new HashMap<>(); result.put("foo", "bar"); doReturn(CompletableFuture.completedFuture(result)).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); int times = 0; table.init(TestRemoteTable.getMockContext()); verify(readFn, times(1)).init(any()); assertEquals("bar", table.getAsync("foo").get()); verify(readFn, times(1)).getAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(result, table.getAllAsync(Arrays.asList("foo")).get()); verify(readFn, times(1)).getAllAsync(any()); assertEquals(++times, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.retryTimer.getSnapshot().getMax()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertNull(table.writeRetryMetrics); table.close(); verify(readFn, times(1)).close(); }
@Test public void testGetWithRetryDisabled() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(false).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, times(1)).getAsync(any()); assertEquals(0, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(0, table.readRetryMetrics.permFailureCount.getCount()); assertEquals(0, table.readRetryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testPutWithPermFailureOnTimeout() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAsync("foo", "bar").get(); fail(); } catch (ExecutionException e) { } verify(writeFn, atLeast(3)).putAsync(any(), any()); assertTrue(table.writeRetryMetrics.retryCount.getCount() >= 3); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(1, table.writeRetryMetrics.permFailureCount.getCount()); assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGetWithPermFailureOnTimeout() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(3)).getAsync(any()); assertTrue(table.readRetryMetrics.retryCount.getCount() >= 3); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testPutWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).putAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAllAsync(Arrays.asList(new Entry(1, 2))).get(); fail(); } catch (ExecutionException e) { } verify(writeFn, atLeast(11)).putAllAsync(any()); assertEquals(10, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(1, table.writeRetryMetrics.permFailureCount.getCount()); assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGetWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); doReturn(true).when(readFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, null); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, policy, null, schedExec, readFn, null); table.init(TestRemoteTable.getMockContext()); try { table.getAsync("foo").get(); fail(); } catch (ExecutionException e) { } verify(readFn, atLeast(11)).getAsync(any()); assertEquals(10, table.readRetryMetrics.retryCount.getCount()); assertEquals(0, table.readRetryMetrics.successCount.getCount()); assertEquals(1, table.readRetryMetrics.permFailureCount.getCount()); assertTrue(table.readRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testGetMaxMinAverageSumSize() { Snapshot snapshot = new Snapshot(Arrays.asList(1L, 2L, 3L, 4L, 5L)); assertEquals(5, snapshot.getMax()); assertEquals(1, snapshot.getMin()); assertEquals(3, snapshot.getAverage(), 0); assertEquals(15, snapshot.getSum(), 0); assertEquals(5, snapshot.getSize()); Snapshot emptySnapshot = new Snapshot(new ArrayList<>()); assertEquals(0, emptySnapshot.getMax()); assertEquals(0, emptySnapshot.getMin()); assertEquals(0, emptySnapshot.getAverage(), 0); assertEquals(0, emptySnapshot.getSum(), 0); assertEquals(0, emptySnapshot.getSize()); } }