private void updateMetricOnAggregateAndHost( CassandraClientPoolingContainer hostPool, Consumer<RequestMetrics> metricsConsumer) { metricsConsumer.accept(aggregateMetrics); RequestMetrics requestMetricsForHost = metricsByHost.get(hostPool.getHost()); if (requestMetricsForHost != null) { metricsConsumer.accept(requestMetricsForHost); } }
/** * The key for a host is the open upper bound of the weight. Since the domain is intended to be contiguous, the * closed lower bound of that weight is the key of the previous entry. * <p> * The closed lower bound of the first entry is 0. * <p> * Every weight is guaranteed to be non-zero in size. That is, every key is guaranteed to be at least one larger * than the previous key. */ private static NavigableMap<Integer, InetSocketAddress> buildHostsWeightedByActiveConnections( Map<InetSocketAddress, CassandraClientPoolingContainer> pools) { Map<InetSocketAddress, Integer> openRequestsByHost = new HashMap<>(pools.size()); int totalOpenRequests = 0; for (Map.Entry<InetSocketAddress, CassandraClientPoolingContainer> poolEntry : pools.entrySet()) { int openRequests = Math.max(poolEntry.getValue().getOpenRequests(), 0); openRequestsByHost.put(poolEntry.getKey(), openRequests); totalOpenRequests += openRequests; } int lowerBoundInclusive = 0; NavigableMap<Integer, InetSocketAddress> weightedHosts = new TreeMap<>(); for (Map.Entry<InetSocketAddress, Integer> entry : openRequestsByHost.entrySet()) { // We want the weight to be inversely proportional to the number of open requests so that we pick // less-active hosts. We add 1 to make sure that all ranges are non-empty int weight = totalOpenRequests - entry.getValue() + 1; weightedHosts.put(lowerBoundInclusive + weight, entry.getKey()); lowerBoundInclusive += weight; } return weightedHosts; }
@Override public <V, K extends Exception> V runWithPooledResource(FunctionCheckedException<CassandraClient, V, K> fn) throws K { final String origName = Thread.currentThread().getName(); Thread.currentThread().setName(origName + " calling cassandra host " + host + " started at " + DateTimeFormatter.ISO_INSTANT.format(Instant.now()) + " - " + count.getAndIncrement()); try { openRequests.getAndIncrement(); return runWithGoodResource(fn); } catch (Throwable t) { log.warn("Error occurred talking to host '{}': {}", SafeArg.of("host", CassandraLogHelper.host(host)), UnsafeArg.of("exception", t.toString())); throw t; } finally { openRequests.getAndDecrement(); Thread.currentThread().setName(origName); } }
public void debugLogStateOfPool() { if (log.isDebugEnabled()) { StringBuilder currentState = new StringBuilder(); currentState.append( String.format("POOL STATUS: Current blacklist = %s,%n current hosts in pool = %s%n", blacklist.describeBlacklistedHosts(), currentPools.keySet().toString())); for (Map.Entry<InetSocketAddress, CassandraClientPoolingContainer> entry : currentPools.entrySet()) { int activeCheckouts = entry.getValue().getActiveCheckouts(); int totalAllowed = entry.getValue().getPoolSize(); currentState.append( String.format("\tPOOL STATUS: Pooled host %s has %s out of %s connections checked out.%n", entry.getKey(), activeCheckouts > 0 ? Integer.toString(activeCheckouts) : "(unknown)", totalAllowed > 0 ? Integer.toString(totalAllowed) : "(not bounded)")); } log.debug("Current pool state: {}", currentState.toString()); } }
private boolean isHostHealthy(CassandraClientPoolingContainer container) { try { container.runWithPooledResource(CassandraUtils.getDescribeRing(config)); container.runWithPooledResource(CassandraUtils.getValidatePartitioner(config)); return true; } catch (Exception e) { log.info("We tried to add blacklisted host '{}' back into the pool, but got an exception" + " that caused us to distrust this host further. Exception message was: {} : {}", SafeArg.of("host", CassandraLogHelper.host(container.getHost())), SafeArg.of("exceptionClass", e.getClass().getCanonicalName()), UnsafeArg.of("exceptionMessage", e.getMessage())); return false; } }
return fn.apply(resource); } catch (Exception e) { if (isInvalidClientConnection(resource)) { log.warn("Not reusing resource {} due to {} of host {}", UnsafeArg.of("resource", resource), UnsafeArg.of("resource", resource), SafeArg.of("host", CassandraLogHelper.host(host))); eagerlyCleanupReadBuffersFromIdleConnection(resource, host); clientPool.returnObject(resource); } else { invalidateQuietly(resource);
private List<TokenRange> getTokenRanges() throws Exception { return getRandomGoodHost().runWithPooledResource(CassandraUtils.getDescribeRing(config)); }
public void addPool(InetSocketAddress server) { int currentPoolNumber = cassandraHosts.indexOf(server) + 1; currentPools.put(server, new CassandraClientPoolingContainer(metricsManager, server, config, currentPoolNumber)); }
public CassandraClientPoolingContainer( MetricsManager metricsManager, InetSocketAddress host, CassandraKeyValueServiceConfig config, int poolNumber) { this.metricsManager = metricsManager; this.host = host; this.config = config; this.poolNumber = poolNumber; this.clientPool = createClientPool(); }
void inPool(CassandraClientPool cassandraClientPool) { CassandraClientPoolingContainer container = mock(CassandraClientPoolingContainer.class); when(container.getHost()).thenReturn(address); try { OngoingStubbing<Object> stubbing = when(container.runWithPooledResource( Mockito.<FunctionCheckedException<CassandraClient, Object, Exception>>any())); for (Exception ex : exceptions) { stubbing = stubbing.thenThrow(ex); } if (returnsValue) { stubbing.thenReturn("Response"); } } catch (Exception ex) { throw new RuntimeException(ex); } cassandraClientPool.getCurrentPools().put(address, container); } }
return fn.apply(resource); } catch (Exception e) { if (isInvalidClientConnection(resource)) { log.warn("Not reusing resource {} due to {} of host {}", UnsafeArg.of("resource", resource), UnsafeArg.of("resource", resource), SafeArg.of("host", CassandraLogHelper.host(host))); eagerlyCleanupReadBuffersFromIdleConnection(resource, host); clientPool.returnObject(resource); } else { invalidateQuietly(resource);
private void setFailureModeForHost(CassandraClientPoolingContainer poolingContainer, Exception failureMode) { try { when(poolingContainer.runWithPooledResource( Mockito.<FunctionCheckedException<CassandraClient, Object, Exception>>any())) .thenThrow(failureMode); } catch (Exception e) { throw Throwables.propagate(e); } }
public void debugLogStateOfPool() { if (log.isDebugEnabled()) { StringBuilder currentState = new StringBuilder(); currentState.append( String.format("POOL STATUS: Current blacklist = %s,%n current hosts in pool = %s%n", blacklist.describeBlacklistedHosts(), currentPools.keySet().toString())); for (Map.Entry<InetSocketAddress, CassandraClientPoolingContainer> entry : currentPools.entrySet()) { int activeCheckouts = entry.getValue().getActiveCheckouts(); int totalAllowed = entry.getValue().getPoolSize(); currentState.append( String.format("\tPOOL STATUS: Pooled host %s has %s out of %s connections checked out.%n", entry.getKey(), activeCheckouts > 0 ? Integer.toString(activeCheckouts) : "(unknown)", totalAllowed > 0 ? Integer.toString(totalAllowed) : "(not bounded)")); } log.debug("Current pool state: {}", currentState.toString()); } }
public void addPool(InetSocketAddress server) { int currentPoolNumber = cassandraHosts.indexOf(server) + 1; currentPools.put(server, new CassandraClientPoolingContainer(metricsManager, server, config, currentPoolNumber)); }
public CassandraClientPoolingContainer( MetricsManager metricsManager, InetSocketAddress host, CassandraKeyValueServiceConfig config, int poolNumber) { this.metricsManager = metricsManager; this.host = host; this.config = config; this.poolNumber = poolNumber; this.clientPool = createClientPool(); }
@Override public <V, K extends Exception> V runWithRetry(FunctionCheckedException<CassandraClient, V, K> fn) throws K { return runWithRetryOnHost(cassandra.getRandomGoodHost().getHost(), fn); }
private boolean isHostHealthy(CassandraClientPoolingContainer container) { try { container.runWithPooledResource(CassandraUtils.getDescribeRing(config)); container.runWithPooledResource(CassandraUtils.getValidatePartitioner(config)); return true; } catch (Exception e) { log.info("We tried to add blacklisted host '{}' back into the pool, but got an exception" + " that caused us to distrust this host further. Exception message was: {} : {}", SafeArg.of("host", CassandraLogHelper.host(container.getHost())), SafeArg.of("exceptionClass", e.getClass().getCanonicalName()), UnsafeArg.of("exceptionMessage", e.getMessage())); return false; } }
private void verifyNumberOfAttemptsOnHost(InetSocketAddress host, CassandraClientPool cassandraClientPool, int numAttempts) { Mockito.verify(cassandraClientPool.getCurrentPools().get(host), Mockito.times(numAttempts)) .runWithPooledResource( Mockito.<FunctionCheckedException<CassandraClient, Object, RuntimeException>>any()); }
private static CassandraClientPoolingContainer createMockClientPoolingContainerWithUtilization(int utilization) { CassandraClientPoolingContainer mock = Mockito.mock(CassandraClientPoolingContainer.class); Mockito.when(mock.getOpenRequests()).thenReturn(utilization); return mock; } }
@Override @SuppressWarnings("unchecked") public <V, K extends Exception> V runWithPooledResource(FunctionCheckedException<Client, V, K> f) throws K { final String origName = Thread.currentThread().getName(); Thread.currentThread().setName(origName + " calling cassandra host " + host + " started at " + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(new Date()) + " - " + count.getAndIncrement()); try { return runWithGoodResource(f); } catch (Throwable t) { log.warn("Failed while connecting to host: " + host, t); if (t instanceof Exception) { throw (K) t; } else { throw (Error) t; } } finally { Thread.currentThread().setName(origName); } }