private void updateMetricOnAggregateAndHost( CassandraClientPoolingContainer hostPool, Consumer<RequestMetrics> metricsConsumer) { metricsConsumer.accept(aggregateMetrics); RequestMetrics requestMetricsForHost = metricsByHost.get(hostPool.getHost()); if (requestMetricsForHost != null) { metricsConsumer.accept(requestMetricsForHost); } }
@Override public <V, K extends Exception> V runWithRetry(FunctionCheckedException<CassandraClient, V, K> fn) throws K { return runWithRetryOnHost(cassandra.getRandomGoodHost().getHost(), fn); }
@Override public <V, K extends Exception> V run(FunctionCheckedException<CassandraClient, V, K> fn) throws K { return runOnHost(cassandra.getRandomGoodHost().getHost(), fn); }
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") // Unpacking it seems less readable private CassandraClientPoolingContainer getMockPoolingContainerForHost(InetSocketAddress address, Optional<Exception> maybeFailureMode) { CassandraClientPoolingContainer poolingContainer = mock(CassandraClientPoolingContainer.class); when(poolingContainer.getHost()).thenReturn(address); maybeFailureMode.ifPresent(e -> setFailureModeForHost(poolingContainer, e)); return poolingContainer; }
@SuppressWarnings({"OptionalUsedAsFieldOrParameterType", "ConstantConditions"}) private void assertContainerHasHostOne(Optional<CassandraClientPoolingContainer> container) { assertThat(container.isPresent(), is(true)); assertThat(container.get().getHost(), equalTo(HOST_1)); }
public InetSocketAddress getRandomHostForKey(byte[] key) { List<InetSocketAddress> hostsForKey = getHostsFor(key); if (hostsForKey == null) { log.info("We attempted to route your query to a cassandra host that already contains the relevant data." + " However, the mapping of which host contains which data is not available yet." + " We will choose a random host instead."); return getRandomGoodHost().getHost(); } Set<InetSocketAddress> liveOwnerHosts = blacklist.filterBlacklistedHostsFrom(hostsForKey); if (!liveOwnerHosts.isEmpty()) { Optional<InetSocketAddress> activeHost = getRandomHostByActiveConnections(liveOwnerHosts); if (activeHost.isPresent()) { return activeHost.get(); } } log.warn("Perf / cluster stability issue. Token aware query routing has failed because there are no known " + "live hosts that claim ownership of the given range. Falling back to choosing a random live node." + " Current host blacklist is {}." + " Current state logged at TRACE", SafeArg.of("blacklistedHosts", blacklist.blacklistDetails())); log.trace("Current ring view is: {}.", SafeArg.of("tokenMap", getRingViewDescription())); return getRandomGoodHost().getHost(); }
private boolean isHostHealthy(CassandraClientPoolingContainer container) { try { container.runWithPooledResource(CassandraUtils.getDescribeRing(config)); container.runWithPooledResource(CassandraUtils.getValidatePartitioner(config)); return true; } catch (Exception e) { log.info("We tried to add blacklisted host '{}' back into the pool, but got an exception" + " that caused us to distrust this host further. Exception message was: {} : {}", SafeArg.of("host", CassandraLogHelper.host(container.getHost())), SafeArg.of("exceptionClass", e.getClass().getCanonicalName()), UnsafeArg.of("exceptionMessage", e.getMessage())); return false; } }
void inPool(CassandraClientPool cassandraClientPool) { CassandraClientPoolingContainer container = mock(CassandraClientPoolingContainer.class); when(container.getHost()).thenReturn(address); try { OngoingStubbing<Object> stubbing = when(container.runWithPooledResource( Mockito.<FunctionCheckedException<CassandraClient, Object, Exception>>any())); for (Exception ex : exceptions) { stubbing = stubbing.thenThrow(ex); } if (returnsValue) { stubbing.thenReturn("Response"); } } catch (Exception ex) { throw new RuntimeException(ex); } cassandraClientPool.getCurrentPools().put(address, container); } }
@Override public <V, K extends Exception> V runWithRetryOnHost( InetSocketAddress specifiedHost, FunctionCheckedException<CassandraClient, V, K> fn) throws K { RetryableCassandraRequest<V, K> req = new RetryableCassandraRequest<>(specifiedHost, fn); while (true) { if (log.isTraceEnabled()) { log.trace("Running function on host {}.", SafeArg.of("host", CassandraLogHelper.host(req.getPreferredHost()))); } CassandraClientPoolingContainer hostPool = getPreferredHostOrFallBack(req); try { V response = runWithPooledResourceRecordingMetrics(hostPool, req.getFunction()); removeFromBlacklistAfterResponse(hostPool.getHost()); return response; } catch (Exception ex) { exceptionHandler.handleExceptionFromRequest(req, hostPool.getHost(), ex); } } }
private <V, K extends Exception> CassandraClientPoolingContainer getPreferredHostOrFallBack( RetryableCassandraRequest<V, K> req) { CassandraClientPoolingContainer hostPool = cassandra.getPools().get(req.getPreferredHost()); if (blacklist.contains(req.getPreferredHost()) || hostPool == null || req.shouldGiveUpOnPreferredHost()) { InetSocketAddress previousHost = hostPool == null ? req.getPreferredHost() : hostPool.getHost(); Optional<CassandraClientPoolingContainer> hostPoolCandidate = cassandra.getRandomGoodHostForPredicate(address -> !req.alreadyTriedOnHost(address)); hostPool = hostPoolCandidate.orElseGet(cassandra::getRandomGoodHost); log.warn("Randomly redirected a query intended for host {} to {}.", SafeArg.of("previousHost", CassandraLogHelper.host(previousHost)), SafeArg.of("randomHost", CassandraLogHelper.host(hostPool.getHost()))); } return hostPool; }
@Test public void resilientToRollingRestarts() { CassandraClientPool cassandraClientPool = clientPoolWithServersInCurrentPool(ImmutableSet.of(HOST_1, HOST_2)); AtomicReference<InetSocketAddress> downHost = new AtomicReference<>(HOST_1); cassandraClientPool.getCurrentPools().values().forEach(pool -> setConditionalTimeoutFailureForHost( pool, container -> container.getHost().equals(downHost.get()))); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(true)); downHost.set(HOST_2); runNoopWithRetryOnHost(HOST_2, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(false)); }
private void updateMetricOnAggregateAndHost( CassandraClientPoolingContainer hostPool, Consumer<RequestMetrics> metricsConsumer) { metricsConsumer.accept(aggregateMetrics); RequestMetrics requestMetricsForHost = metricsByHost.get(hostPool.getHost()); if (requestMetricsForHost != null) { metricsConsumer.accept(requestMetricsForHost); } }
@Override public <V, K extends Exception> V runWithRetry(FunctionCheckedException<CassandraClient, V, K> fn) throws K { return runWithRetryOnHost(cassandra.getRandomGoodHost().getHost(), fn); }
@Override public <V, K extends Exception> V run(FunctionCheckedException<CassandraClient, V, K> fn) throws K { return runOnHost(cassandra.getRandomGoodHost().getHost(), fn); }
public InetSocketAddress getRandomHostForKey(byte[] key) { List<InetSocketAddress> hostsForKey = getHostsFor(key); if (hostsForKey == null) { log.info("We attempted to route your query to a cassandra host that already contains the relevant data." + " However, the mapping of which host contains which data is not available yet." + " We will choose a random host instead."); return getRandomGoodHost().getHost(); } Set<InetSocketAddress> liveOwnerHosts = blacklist.filterBlacklistedHostsFrom(hostsForKey); if (!liveOwnerHosts.isEmpty()) { Optional<InetSocketAddress> activeHost = getRandomHostByActiveConnections(liveOwnerHosts); if (activeHost.isPresent()) { return activeHost.get(); } } log.warn("Perf / cluster stability issue. Token aware query routing has failed because there are no known " + "live hosts that claim ownership of the given range. Falling back to choosing a random live node." + " Current host blacklist is {}." + " Current state logged at TRACE", SafeArg.of("blacklistedHosts", blacklist.blacklistDetails())); log.trace("Current ring view is: {}.", SafeArg.of("tokenMap", getRingViewDescription())); return getRandomGoodHost().getHost(); }
private boolean isHostHealthy(CassandraClientPoolingContainer container) { try { container.runWithPooledResource(CassandraUtils.getDescribeRing(config)); container.runWithPooledResource(CassandraUtils.getValidatePartitioner(config)); return true; } catch (Exception e) { log.info("We tried to add blacklisted host '{}' back into the pool, but got an exception" + " that caused us to distrust this host further. Exception message was: {} : {}", SafeArg.of("host", CassandraLogHelper.host(container.getHost())), SafeArg.of("exceptionClass", e.getClass().getCanonicalName()), UnsafeArg.of("exceptionMessage", e.getMessage())); return false; } }
private <V, K extends Exception> CassandraClientPoolingContainer getPreferredHostOrFallBack( RetryableCassandraRequest<V, K> req) { CassandraClientPoolingContainer hostPool = cassandra.getPools().get(req.getPreferredHost()); if (blacklist.contains(req.getPreferredHost()) || hostPool == null || req.shouldGiveUpOnPreferredHost()) { InetSocketAddress previousHost = hostPool == null ? req.getPreferredHost() : hostPool.getHost(); Optional<CassandraClientPoolingContainer> hostPoolCandidate = cassandra.getRandomGoodHostForPredicate(address -> !req.alreadyTriedOnHost(address)); hostPool = hostPoolCandidate.orElseGet(cassandra::getRandomGoodHost); log.warn("Randomly redirected a query intended for host {} to {}.", SafeArg.of("previousHost", CassandraLogHelper.host(previousHost)), SafeArg.of("randomHost", CassandraLogHelper.host(hostPool.getHost()))); } return hostPool; }
@Override public <V, K extends Exception> V runWithRetryOnHost( InetSocketAddress specifiedHost, FunctionCheckedException<CassandraClient, V, K> fn) throws K { RetryableCassandraRequest<V, K> req = new RetryableCassandraRequest<>(specifiedHost, fn); while (true) { if (log.isTraceEnabled()) { log.trace("Running function on host {}.", SafeArg.of("host", CassandraLogHelper.host(req.getPreferredHost()))); } CassandraClientPoolingContainer hostPool = getPreferredHostOrFallBack(req); try { V response = runWithPooledResourceRecordingMetrics(hostPool, req.getFunction()); removeFromBlacklistAfterResponse(hostPool.getHost()); return response; } catch (Exception ex) { exceptionHandler.handleExceptionFromRequest(req, hostPool.getHost(), ex); } } }