private <T> Predicate<T> getMutationRetryableFunction(Predicate<T> isRetryableMutation) { if (retryOptions.allowRetriesWithoutTimestamp()) { return new Predicate<T>() { @Override public boolean apply(@Nullable T input) { return input != null; } }; } else { return isRetryableMutation; } }
public static RetryOptions getDefaultOptions() { return builder().build(); }
@Deprecated public Builder() { options = new RetryOptions(); options.retriesEnabled = DEFAULT_ENABLE_GRPC_RETRIES; options.allowRetriesWithoutTimestamp = false; options.initialBackoffMillis = DEFAULT_INITIAL_BACKOFF_MILLIS; options.maxElapsedBackoffMillis = DEFAULT_MAX_ELAPSED_BACKOFF_MILLIS; options.backoffMultiplier = DEFAULT_BACKOFF_MULTIPLIER; options.streamingBufferSize = DEFAULT_STREAMING_BUFFER_SIZE; options.readPartialRowTimeoutMillis = DEFAULT_READ_PARTIAL_ROW_TIMEOUT_MS; options.maxScanTimeoutRetries = DEFAULT_MAX_SCAN_TIMEOUT_RETRIES; options.statusToRetryOn = new HashSet<>(DEFAULT_ENABLE_GRPC_RETRIES_SET); }
options.dataHost = BIGTABLE_BATCH_DATA_HOST_DEFAULT; RetryOptions.Builder retryOptionsBuilder = options.retryOptions.toBuilder(); if (options.retryOptions.getInitialBackoffMillis() == RetryOptions.DEFAULT_INITIAL_BACKOFF_MILLIS) { retryOptionsBuilder.setInitialBackoffMillis((int) TimeUnit.SECONDS.toMillis(5)); if (options.retryOptions.getMaxElapsedBackoffMillis() == RetryOptions.DEFAULT_MAX_ELAPSED_BACKOFF_MILLIS) { retryOptionsBuilder.setMaxElapsedBackoffMillis((int) TimeUnit.MINUTES.toMillis(5));
/** * <p>createBackoff.</p> * * @return a {@link ExponentialRetryAlgorithm} object. */ private ExponentialRetryAlgorithm createRetryAlgorithm(ApiClock clock) { long timeoutMs = retryOptions.getMaxElapsedBackoffMillis(); Deadline deadline = getOperationCallOptions().getDeadline(); if (deadline != null) { timeoutMs = deadline.timeRemaining(TimeUnit.MILLISECONDS); } RetrySettings retrySettings = RetrySettings.newBuilder() .setJittered(true) // How long should the sleep be between RPC failure and the next RPC retry? .setInitialRetryDelay(toDuration(retryOptions.getInitialBackoffMillis())) // How fast should the retry delay increase? .setRetryDelayMultiplier(retryOptions.getBackoffMultiplier()) // What is the maximum amount of sleep time between retries? // There needs to be some sane number for max retry delay, and it's unclear what that // number ought to be. 1 Minute time was chosen because some number is needed. .setMaxRetryDelay(Duration.of(1, ChronoUnit.MINUTES)) // How long should we wait before giving up retries after the first failure? .setTotalTimeout(toDuration(timeoutMs)) .build(); return new ExponentialRetryAlgorithm(retrySettings, clock); }
@Test public void testReadWithBigTableOptionsSetsRetryOptions() { final int initialBackoffMillis = -1; BigtableOptions.Builder optionsBuilder = BIGTABLE_OPTIONS.toBuilder(); RetryOptions.Builder retryOptionsBuilder = new RetryOptions.Builder(); retryOptionsBuilder.setInitialBackoffMillis(initialBackoffMillis); optionsBuilder.setRetryOptions(retryOptionsBuilder.build()); BigtableIO.Read read = BigtableIO.read().withBigtableOptions(optionsBuilder.build()); BigtableOptions options = read.getBigtableOptions(); assertEquals(initialBackoffMillis, options.getRetryOptions().getInitialBackoffMillis()); assertThat(options.getRetryOptions(), Matchers.equalTo(retryOptionsBuilder.build())); }
/** * Special retry handling for watchdog timeouts, which uses its own fail counter. * * @return true if a retry has been scheduled */ private void handleTimeoutError(Status status) { Preconditions.checkArgument(status.getCause() instanceof StreamWaitTimeoutException, "status is not caused by a StreamWaitTimeoutException"); StreamWaitTimeoutException e = ((StreamWaitTimeoutException) status.getCause()); // Cancel the existing rpc. rpcTimerContext.close(); failedCount++; // Can this request be retried int maxRetries = retryOptions.getMaxScanTimeoutRetries(); if (retryOptions.enableRetries() && ++timeoutRetryCount <= maxRetries) { LOG.warn("The client could not get a response in %d ms. Retrying the scan.", e.getWaitTimeMs()); resetStatusBasedBackoff(); performRetry(0); } else { LOG.warn("The client could not get a response after %d tries, giving up.", timeoutRetryCount); rpc.getRpcMetrics().markFailure(); finalizeStats(status); setException(getExhaustedRetriesException(status)); } }
if (!retryOptions.enableRetries() || !retryOptions.isRetryable(code)
public Builder(RetryOptions options) { this.options = options.clone(); }
private WatchdogInterceptor setupWatchdog() { Preconditions.checkState(watchdog == null, "Watchdog already setup"); watchdog = new Watchdog(Clock.SYSTEM, options.getRetryOptions().getReadPartialRowTimeoutMillis()); watchdog.start(BigtableSessionSharedThreadPools.getInstance().getRetryExecutor()); return new WatchdogInterceptor( ImmutableSet.<MethodDescriptor<?, ?>>of(BigtableGrpc.getReadRowsMethod()), watchdog); }
@Test public void testWriteWithBigTableOptionsSetsBulkOptionsAndRetryOptions() { final int maxInflightRpcs = 1; final int initialBackoffMillis = -1; BigtableOptions.Builder optionsBuilder = BIGTABLE_OPTIONS.toBuilder(); BulkOptions.Builder bulkOptionsBuilder = new BulkOptions.Builder(); bulkOptionsBuilder.setMaxInflightRpcs(maxInflightRpcs); RetryOptions.Builder retryOptionsBuilder = new RetryOptions.Builder(); retryOptionsBuilder.setInitialBackoffMillis(initialBackoffMillis); optionsBuilder .setBulkOptions(bulkOptionsBuilder.build()) .setRetryOptions(retryOptionsBuilder.build()); BigtableIO.Write write = BigtableIO.write().withBigtableOptions(optionsBuilder.build()); BigtableOptions options = write.getBigtableOptions(); assertEquals(true, options.getBulkOptions().useBulkApi()); assertEquals(maxInflightRpcs, options.getBulkOptions().getMaxInflightRpcs()); assertEquals(initialBackoffMillis, options.getRetryOptions().getInitialBackoffMillis()); assertThat( options.getBulkOptions(), Matchers.equalTo(bulkOptionsBuilder.setUseBulkApi(true).build())); assertThat(options.getRetryOptions(), Matchers.equalTo(retryOptionsBuilder.build())); }
/** * <p>createPutAdapter.</p> * * @param config a {@link org.apache.hadoop.conf.Configuration} object. * @param options a {@link com.google.cloud.bigtable.config.BigtableOptions} object. * @return a {@link com.google.cloud.bigtable.hbase.adapters.PutAdapter} object. */ public static PutAdapter createPutAdapter(Configuration config, BigtableOptions options) { boolean setClientTimestamp = !options.getRetryOptions().allowRetriesWithoutTimestamp(); return new PutAdapter(config.getInt("hbase.client.keyvalue.maxsize", -1), setClientTimestamp); }
private static RetryOptions createRetryOptions(Configuration configuration) { RetryOptions.Builder retryOptionsBuilder = RetryOptions.builder(); boolean enableRetries = configuration.getBoolean( ENABLE_GRPC_RETRIES_KEY, RetryOptions.DEFAULT_ENABLE_GRPC_RETRIES);
/** * <p>createPutAdapter.</p> * * @param config a {@link org.apache.hadoop.conf.Configuration} object. * @param options a {@link com.google.cloud.bigtable.config.BigtableOptions} object. * @return a {@link com.google.cloud.bigtable.hbase.adapters.PutAdapter} object. */ public static PutAdapter createPutAdapter(Configuration config, BigtableOptions options) { boolean setClientTimestamp = !options.getRetryOptions().allowRetriesWithoutTimestamp(); return new PutAdapter(config.getInt("hbase.client.keyvalue.maxsize", -1), setClientTimestamp); }
private static RetryOptions createRetryOptions(Configuration configuration) { RetryOptions.Builder retryOptionsBuilder = RetryOptions.builder(); boolean enableRetries = configuration.getBoolean( ENABLE_GRPC_RETRIES_KEY, RetryOptions.DEFAULT_ENABLE_GRPC_RETRIES);