public static CassandraKeyValueService createForTesting( CassandraKeyValueServiceConfig config, Optional<LeaderConfig> leaderConfig) { MetricsManager metricsManager = MetricsManagers.createForTests(); CassandraClientPool clientPool = CassandraClientPoolImpl.createImplForTest(metricsManager, config, CassandraClientPoolImpl.StartupChecks.RUN, new Blacklist(config)); return createOrShutdownClientPool(metricsManager, config, clientPool, leaderConfig, CassandraMutationTimestampProviders.legacyModeForTestsOnly(), LoggerFactory.getLogger(CassandraKeyValueService.class), AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC); }
@VisibleForTesting static CassandraClientPoolImpl createImplForTest( MetricsManager metricsManager, CassandraKeyValueServiceConfig config, StartupChecks startupChecks, Blacklist blacklist) { CassandraRequestExceptionHandler exceptionHandler = testExceptionHandler(blacklist); CassandraClientPoolImpl cassandraClientPool = new CassandraClientPoolImpl( metricsManager, config, startupChecks, exceptionHandler, blacklist); cassandraClientPool.wrapper.initialize(AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC); return cassandraClientPool; }
private static CassandraKeyValueService create( MetricsManager metricsManager, CassandraKeyValueServiceConfig config, java.util.function.Supplier<CassandraKeyValueServiceRuntimeConfig> runtimeConfig, Optional<LeaderConfig> leaderConfig, CassandraMutationTimestampProvider mutationTimestampProvider, Logger log, boolean initializeAsync) { CassandraClientPool clientPool = CassandraClientPoolImpl.create(metricsManager, config, runtimeConfig, initializeAsync); return createOrShutdownClientPool(metricsManager, config, clientPool, leaderConfig, mutationTimestampProvider, log, initializeAsync); }
@Override public <V, K extends Exception> V runOnHost(InetSocketAddress specifiedHost, FunctionCheckedException<CassandraClient, V, K> fn) throws K { CassandraClientPoolingContainer hostPool = cassandra.getPools().get(specifiedHost); V response = runWithPooledResourceRecordingMetrics(hostPool, fn); removeFromBlacklistAfterResponse(specifiedHost); return response; }
private void tryInitialize() { cassandra.cacheInitialCassandraHosts(); refreshPoolFuture = refreshDaemon.scheduleWithFixedDelay(() -> { try { refreshPool(); } catch (Throwable t) { log.warn("Failed to refresh Cassandra KVS pool." + " Extended periods of being unable to refresh will cause perf degradation.", t); } }, config.poolRefreshIntervalSeconds(), config.poolRefreshIntervalSeconds(), TimeUnit.SECONDS); // for testability, mock/spy are bad at mockability of things called in constructors if (startupChecks == StartupChecks.RUN) { runOneTimeStartupChecks(); } refreshPool(); // ensure we've initialized before returning metrics.registerAggregateMetrics(blacklist::size); }
@Override public <V, K extends Exception> V runWithRetryOnHost( InetSocketAddress specifiedHost, FunctionCheckedException<CassandraClient, V, K> fn) throws K { RetryableCassandraRequest<V, K> req = new RetryableCassandraRequest<>(specifiedHost, fn); while (true) { if (log.isTraceEnabled()) { log.trace("Running function on host {}.", SafeArg.of("host", CassandraLogHelper.host(req.getPreferredHost()))); } CassandraClientPoolingContainer hostPool = getPreferredHostOrFallBack(req); try { V response = runWithPooledResourceRecordingMetrics(hostPool, req.getFunction()); removeFromBlacklistAfterResponse(hostPool.getHost()); return response; } catch (Exception ex) { exceptionHandler.handleExceptionFromRequest(req, hostPool.getHost(), ex); } } }
boolean thisHostResponded = false; boolean atLeastOneHostResponded = false; for (InetSocketAddress host : getCachedServers()) { thisHostResponded = false; try { runOnHost(host, CassandraVerifier.healthCheck); thisHostResponded = true; atLeastOneHostResponded = true; runOnHost(host, getValidatePartitioner()); } catch (Exception e) { aliveButInvalidPartitionerHosts.put(host, e);
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") // Unpacking it seems less readable private CassandraClientPoolImpl clientPoolWith( ImmutableSet<InetSocketAddress> servers, ImmutableSet<InetSocketAddress> serversInPool, Optional<Exception> failureMode) { when(config.servers()).thenReturn(servers); CassandraClientPoolImpl cassandraClientPool = CassandraClientPoolImpl.createImplForTest( MetricsManagers.of(metricRegistry, taggedMetricRegistry), config, CassandraClientPoolImpl.StartupChecks.DO_NOT_RUN, blacklist); serversInPool.forEach(address -> cassandraClientPool.getCurrentPools() .put(address, getMockPoolingContainerForHost(address, failureMode))); return cassandraClientPool; }
private void setServersInPoolTo(Set<InetSocketAddress> desiredServers) { Set<InetSocketAddress> serversToRemove = Sets.newHashSet(); Set<InetSocketAddress> serversToAdd = Sets.newHashSet(); Set<InetSocketAddress> cachedServers = getCachedServers(); serversToAdd.addAll(Sets.difference(desiredServers, cachedServers)); serversToRemove.addAll(Sets.difference(cachedServers, desiredServers)); serversToAdd.forEach(cassandra::addPool); serversToRemove.forEach(cassandra::removePool); if (!(serversToAdd.isEmpty() && serversToRemove.isEmpty())) { // if we made any changes sanityCheckRingConsistency(); cassandra.refreshTokenRangesAndGetServers(); } log.debug("Cassandra pool refresh added hosts {}, removed hosts {}.", SafeArg.of("serversToAdd", CassandraLogHelper.collectionOfHosts(serversToAdd)), SafeArg.of("serversToRemove", CassandraLogHelper.collectionOfHosts(serversToRemove))); }
@Test public void testPoolGivenNoOptionTalksToBlacklistedHosts() { blacklist.addAll(clientPool.getCurrentPools().keySet()); try { clientPool.run(describeRing); } catch (Exception e) { fail("Should have been allowed to attempt forward progress after blacklisting all hosts in pool."); } blacklist.removeAll(); }
@Override public <V, K extends Exception> V run(FunctionCheckedException<CassandraClient, V, K> fn) throws K { return runOnHost(cassandra.getRandomGoodHost().getHost(), fn); }
public static CassandraClientPool create(MetricsManager metricsManager, CassandraKeyValueServiceConfig config, Supplier<CassandraKeyValueServiceRuntimeConfig> runtimeConfig, boolean initializeAsync) { Blacklist blacklist = new Blacklist(config); CassandraRequestExceptionHandler exceptionHandler = new CassandraRequestExceptionHandler( () -> runtimeConfig.get().numberOfRetriesOnSameHost(), () -> runtimeConfig.get().numberOfRetriesOnAllHosts(), () -> runtimeConfig.get().conservativeRequestExceptionHandler(), blacklist); CassandraClientPoolImpl cassandraClientPool = new CassandraClientPoolImpl( metricsManager, config, StartupChecks.RUN, exceptionHandler, blacklist); cassandraClientPool.wrapper.initialize(initializeAsync); return cassandraClientPool.wrapper.isInitialized() ? cassandraClientPool : cassandraClientPool.wrapper; }
private void sanityCheckRingConsistency() { Multimap<Set<TokenRange>, InetSocketAddress> tokenRangesToHost = HashMultimap.create(); for (InetSocketAddress host : getCachedServers()) { CassandraClient client = null; try {
private void runTwoNoopsOnTwoHostsAndThrowFromSecondRunOnFirstHost(Exception exception) { CassandraClientPoolImpl cassandraClientPool = clientPoolWithServersInCurrentPool( ImmutableSet.of(HOST_1, HOST_2)); runNoopOnHost(HOST_1, cassandraClientPool); runNoopOnHost(HOST_2, cassandraClientPool); runNoopOnHost(HOST_2, cassandraClientPool); CassandraClientPoolingContainer container = cassandraClientPool.getCurrentPools().get(HOST_1); setFailureModeForHost(container, exception); assertThatThrownBy(() -> runNoopOnHost(HOST_1, cassandraClientPool)).isInstanceOf(Exception.class); }
@Override public <V, K extends Exception> V runWithRetryOnHost( InetSocketAddress specifiedHost, FunctionCheckedException<CassandraClient, V, K> fn) throws K { RetryableCassandraRequest<V, K> req = new RetryableCassandraRequest<>(specifiedHost, fn); while (true) { if (log.isTraceEnabled()) { log.trace("Running function on host {}.", SafeArg.of("host", CassandraLogHelper.host(req.getPreferredHost()))); } CassandraClientPoolingContainer hostPool = getPreferredHostOrFallBack(req); try { V response = runWithPooledResourceRecordingMetrics(hostPool, req.getFunction()); removeFromBlacklistAfterResponse(hostPool.getHost()); return response; } catch (Exception ex) { exceptionHandler.handleExceptionFromRequest(req, hostPool.getHost(), ex); } } }
boolean thisHostResponded = false; boolean atLeastOneHostResponded = false; for (InetSocketAddress host : getCachedServers()) { thisHostResponded = false; try { runOnHost(host, CassandraVerifier.healthCheck); thisHostResponded = true; atLeastOneHostResponded = true; runOnHost(host, getValidatePartitioner()); } catch (Exception e) { aliveButInvalidPartitionerHosts.put(host, e);
@Override public <V, K extends Exception> V runOnHost(InetSocketAddress specifiedHost, FunctionCheckedException<CassandraClient, V, K> fn) throws K { CassandraClientPoolingContainer hostPool = cassandra.getPools().get(specifiedHost); V response = runWithPooledResourceRecordingMetrics(hostPool, fn); removeFromBlacklistAfterResponse(specifiedHost); return response; }
private void setServersInPoolTo(Set<InetSocketAddress> desiredServers) { Set<InetSocketAddress> serversToRemove = Sets.newHashSet(); Set<InetSocketAddress> serversToAdd = Sets.newHashSet(); Set<InetSocketAddress> cachedServers = getCachedServers(); serversToAdd.addAll(Sets.difference(desiredServers, cachedServers)); serversToRemove.addAll(Sets.difference(cachedServers, desiredServers)); serversToAdd.forEach(cassandra::addPool); serversToRemove.forEach(cassandra::removePool); if (!(serversToAdd.isEmpty() && serversToRemove.isEmpty())) { // if we made any changes sanityCheckRingConsistency(); cassandra.refreshTokenRangesAndGetServers(); } log.debug("Cassandra pool refresh added hosts {}, removed hosts {}.", SafeArg.of("serversToAdd", CassandraLogHelper.collectionOfHosts(serversToAdd)), SafeArg.of("serversToRemove", CassandraLogHelper.collectionOfHosts(serversToRemove))); }
private void tryInitialize() { cassandra.cacheInitialCassandraHosts(); refreshPoolFuture = refreshDaemon.scheduleWithFixedDelay(() -> { try { refreshPool(); } catch (Throwable t) { log.warn("Failed to refresh Cassandra KVS pool." + " Extended periods of being unable to refresh will cause perf degradation.", t); } }, config.poolRefreshIntervalSeconds(), config.poolRefreshIntervalSeconds(), TimeUnit.SECONDS); // for testability, mock/spy are bad at mockability of things called in constructors if (startupChecks == StartupChecks.RUN) { runOneTimeStartupChecks(); } refreshPool(); // ensure we've initialized before returning metrics.registerAggregateMetrics(blacklist::size); }
@Override public <V, K extends Exception> V run(FunctionCheckedException<CassandraClient, V, K> fn) throws K { return runOnHost(cassandra.getRandomGoodHost().getHost(), fn); }