@Override public boolean isRetriable(Throwable exception) { return writeFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return writeFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return writeFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return writeFn.isRetriable(exception); }
@Override public boolean isRetriable(Throwable exception) { return writeFn.isRetriable(exception); }
public RetriableWriteFunction(TableRetryPolicy policy, TableWriteFunction<K, V> writeFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(writeFn); Preconditions.checkNotNull(retryExecutor); this.writeFn = writeFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> writeFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableWriteFunction(TableRetryPolicy policy, TableWriteFunction<K, V> writeFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(writeFn); Preconditions.checkNotNull(retryExecutor); this.writeFn = writeFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> writeFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableWriteFunction(TableRetryPolicy policy, TableWriteFunction<K, V> writeFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(writeFn); Preconditions.checkNotNull(retryExecutor); this.writeFn = writeFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> writeFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableWriteFunction(TableRetryPolicy policy, TableWriteFunction<K, V> writeFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(writeFn); Preconditions.checkNotNull(retryExecutor); this.writeFn = writeFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> writeFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public RetriableWriteFunction(TableRetryPolicy policy, TableWriteFunction<K, V> writeFn, ScheduledExecutorService retryExecutor) { Preconditions.checkNotNull(policy); Preconditions.checkNotNull(writeFn); Preconditions.checkNotNull(retryExecutor); this.writeFn = writeFn; this.retryExecutor = retryExecutor; Predicate<Throwable> retryPredicate = policy.getRetryPredicate(); policy.withRetryPredicate((ex) -> writeFn.isRetriable(ex) || retryPredicate.test(ex)); this.retryPolicy = FailsafeAdapter.valueOf(policy); }
public AsyncRetriableTable(String tableId, AsyncReadWriteTable<K, V> table, TableRetryPolicy readRetryPolicy, TableRetryPolicy writeRetryPolicy, ScheduledExecutorService retryExecutor, TableReadFunction readFn, TableWriteFunction writeFn) { Preconditions.checkNotNull(tableId, "null tableId"); Preconditions.checkNotNull(table, "null table"); Preconditions.checkNotNull(retryExecutor, "null retryExecutor"); Preconditions.checkArgument(readRetryPolicy != null || writeRetryPolicy != null, "both readRetryPolicy and writeRetryPolicy are null"); this.tableId = tableId; this.table = table; this.retryExecutor = retryExecutor; if (readRetryPolicy != null && readFn != null) { Predicate<Throwable> readRetryPredicate = readRetryPolicy.getRetryPredicate(); readRetryPolicy.withRetryPredicate((ex) -> readFn.isRetriable(ex) || readRetryPredicate.test(ex)); this.readRetryPolicy = FailsafeAdapter.valueOf(readRetryPolicy); } else { this.readRetryPolicy = null; } if (writeRetryPolicy != null && writeFn != null) { Predicate<Throwable> writeRetryPredicate = writeRetryPolicy.getRetryPredicate(); writeRetryPolicy.withRetryPredicate((ex) -> writeFn.isRetriable(ex) || writeRetryPredicate.test(ex)); this.writeRetryPolicy = FailsafeAdapter.valueOf(writeRetryPolicy); } else { this.writeRetryPolicy = null; } }
@Test public void testFirstTimeSuccessPut() throws Exception { String tableId = "testFirstTimeSuccessPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(100)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture("bar")).when(writeFn).putAsync(anyString(), anyString()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); retryIO.putAsync("foo", "bar").get(); verify(writeFn, times(1)).putAsync(anyString(), anyString()); Assert.assertEquals(0, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(1, retryIO.retryMetrics.successCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.retryTimer.getSnapshot().getMax()); }
@Test public void testRetryExhaustedAttemptsPut() throws Exception { String tableId = "testRetryExhaustedAttemptsPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).deleteAllAsync(any()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.deleteAllAsync(Arrays.asList("foo1", "foo2")).get(); Assert.fail(); } catch (ExecutionException e) { } // 1 initial try + 10 retries verify(writeFn, times(11)).deleteAllAsync(any()); Assert.assertEquals(10, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryEngagedPut() throws Exception { String tableId = "testRetryEngagedPut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAllAsync(any()); doReturn(true).when(writeFn).isRetriable(any()); int [] times = new int[] {0}; List<Entry<String, String>> records = new ArrayList<>(); records.add(new Entry<>("foo1", "bar1")); records.add(new Entry<>("foo2", "bar2")); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times[0] > 0) { future.complete(null); } else { times[0]++; future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(writeFn).putAllAsync(any()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); retryIO.putAllAsync(records).get(); verify(writeFn, times(2)).putAllAsync(any()); Assert.assertEquals(1, retryIO.retryMetrics.retryCount.getCount()); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testRetryExhaustedTimePut() throws Exception { String tableId = "testRetryExhaustedTimePut"; TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).deleteAsync(anyString()); RetriableWriteFunction<String, String> retryIO = new RetriableWriteFunction<>(policy, writeFn, schedExec); retryIO.setMetrics(getMetricsUtil(tableId)); try { retryIO.deleteAsync("foo").get(); Assert.fail(); } catch (ExecutionException e) { } // Conservatively: must be at least 3 attempts with 5ms backoff and 100ms maxDelay verify(writeFn, atLeast(3)).deleteAsync(anyString()); Assert.assertTrue(retryIO.retryMetrics.retryCount.getCount() >= 3); Assert.assertEquals(0, retryIO.retryMetrics.successCount.getCount()); Assert.assertTrue(retryIO.retryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testPutAllWithOneRetry() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); AtomicInteger times = new AtomicInteger(); doAnswer(invocation -> { CompletableFuture<Map<String, String>> future = new CompletableFuture(); if (times.get() > 0) { future.complete(null); } else { times.incrementAndGet(); future.completeExceptionally(new RuntimeException("test exception")); } return future; }).when(writeFn).putAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); table.putAllAsync(Arrays.asList(new Entry(1, 2))).get(); verify(writeFn, times(2)).putAllAsync(any()); assertEquals(1, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount()); assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testPutWithRetryDisabled() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(10)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(false).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).putAsync(any(), any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAsync("foo", "bar").get(); fail(); } catch (ExecutionException e) { } verify(writeFn, times(1)).putAsync(any(), any()); assertEquals(0, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(0, table.writeRetryMetrics.permFailureCount.getCount()); assertEquals(0, table.writeRetryMetrics.retryTimer.getSnapshot().getMax()); }
TableReadFunction readFn = mock(TableReadFunction.class); TableWriteFunction writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAsync(any(), any()); doReturn(CompletableFuture.completedFuture(null)).when(writeFn).putAllAsync(any());
@Test public void testPutWithPermFailureOnTimeout() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterDelay(Duration.ofMillis(100)); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(readFn).getAsync(anyString()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAsync("foo", "bar").get(); fail(); } catch (ExecutionException e) { } verify(writeFn, atLeast(3)).putAsync(any(), any()); assertTrue(table.writeRetryMetrics.retryCount.getCount() >= 3); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(1, table.writeRetryMetrics.permFailureCount.getCount()); assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }
@Test public void testPutWithPermFailureOnMaxCount() throws Exception { TableRetryPolicy policy = new TableRetryPolicy(); policy.withFixedBackoff(Duration.ofMillis(5)); policy.withStopAfterAttempts(10); TableReadFunction<String, String> readFn = mock(TableReadFunction.class); TableWriteFunction<String, String> writeFn = mock(TableWriteFunction.class); doReturn(true).when(writeFn).isRetriable(any()); CompletableFuture<String> future = new CompletableFuture(); future.completeExceptionally(new RuntimeException("test exception")); doReturn(future).when(writeFn).putAllAsync(any()); AsyncReadWriteTable delegate = new AsyncRemoteTable(readFn, writeFn); AsyncRetriableTable table = new AsyncRetriableTable("t1", delegate, null, policy, schedExec, readFn, writeFn); table.init(TestRemoteTable.getMockContext()); try { table.putAllAsync(Arrays.asList(new Entry(1, 2))).get(); fail(); } catch (ExecutionException e) { } verify(writeFn, atLeast(11)).putAllAsync(any()); assertEquals(10, table.writeRetryMetrics.retryCount.getCount()); assertEquals(0, table.writeRetryMetrics.successCount.getCount()); assertEquals(1, table.writeRetryMetrics.permFailureCount.getCount()); assertTrue(table.writeRetryMetrics.retryTimer.getSnapshot().getMax() > 0); }