@Override public CircuitBreakerStats stats(String name) { CircuitBreaker breaker = this.breakers.get(name); return new CircuitBreakerStats(breaker.getName(), breaker.getLimit(), breaker.getUsed(), breaker.getOverhead(), breaker.getTrippedCount()); }
breaker.addEstimateBytesAndMaybeBreak(delta, "<reused_arrays>"); } catch (CircuitBreakingException e) { if (isDataAlreadyCreated) { breaker.addWithoutBreaking(delta); breaker.addWithoutBreaking(delta); breaker.addWithoutBreaking(delta);
@Override public AllCircuitBreakerStats stats() { long parentEstimated = 0; List<CircuitBreakerStats> allStats = new ArrayList<>(this.breakers.size()); // Gather the "estimated" count for the parent breaker by adding the // estimations for each individual breaker for (CircuitBreaker breaker : this.breakers.values()) { allStats.add(stats(breaker.getName())); parentEstimated += breaker.getUsed(); } // Manually add the parent breaker settings since they aren't part of the breaker map allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(), parentEstimated, 1.0, parentTripCount.get())); return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()])); }
private void close() { // attempt to close once atomically if (closed.compareAndSet(false, true) == false) { throw new IllegalStateException("Channel is already closed"); } inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-contentLength); }
/** * Checks whether the parent breaker has been tripped */ public void checkParentLimit(String label) throws CircuitBreakingException { long totalUsed = 0; for (CircuitBreaker breaker : this.breakers.values()) { totalUsed += (breaker.getUsed() * breaker.getOverhead()); } long parentLimit = this.parentSettings.getLimit(); if (totalUsed > parentLimit) { this.parentTripCount.incrementAndGet(); final StringBuilder message = new StringBuilder("[parent] Data too large, data for [" + label + "]" + " would be [" + totalUsed + "/" + new ByteSizeValue(totalUsed) + "]" + ", which is larger than the limit of [" + parentLimit + "/" + new ByteSizeValue(parentLimit) + "]"); message.append(", usages ["); message.append(String.join(", ", this.breakers.entrySet().stream().map(e -> { final CircuitBreaker breaker = e.getValue(); final long breakerUsed = (long)(breaker.getUsed() * breaker.getOverhead()); return e.getKey() + "=" + breakerUsed + "/" + new ByteSizeValue(breakerUsed); }) .collect(Collectors.toList()))); message.append("]"); throw new CircuitBreakingException(message.toString(), totalUsed, parentLimit); } }
/** * Flush the {@code flushBuffer} to the breaker, incrementing the total * bytes and resetting the buffer. */ public void flush() { breaker.addEstimateBytesAndMaybeBreak(this.flushBuffer, this.fieldName); this.totalBytes += this.flushBuffer; this.flushBuffer = 0; }
private void assertRequestsFinished() { if (size() > 0) { for (NodeAndClient nodeAndClient : nodes.values()) { CircuitBreaker inFlightRequestsBreaker = getInstance(CircuitBreakerService.class, nodeAndClient.name) .getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS); try { // see #ensureEstimatedStats() assertBusy(() -> { // ensure that our size accounting on transport level is reset properly long bytesUsed = inFlightRequestsBreaker.getUsed(); assertThat("All incoming requests on node [" + nodeAndClient.name + "] should have finished. Expected 0 but got " + bytesUsed, bytesUsed, equalTo(0L)); }); } catch (Exception e) { logger.error("Could not assert finished requests within timeout", e); fail("Could not assert finished requests within timeout on node [" + nodeAndClient.name + "]"); } } } } }
@Override public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) { assert sizeInBytes >= 0 : "When reducing circuit breaker, it should be adjusted with a number higher or " + "equal to 0 and not [" + sizeInBytes + "]"; circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes); } });
@Override public AllCircuitBreakerStats stats() { long parentEstimated = 0; List<CircuitBreakerStats> allStats = new ArrayList<>(this.breakers.size()); // Gather the "estimated" count for the parent breaker by adding the // estimations for each individual breaker for (CircuitBreaker breaker : this.breakers.values()) { allStats.add(stats(breaker.getName())); parentEstimated += breaker.getUsed(); } // Manually add the parent breaker settings since they aren't part of the breaker map allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(), parentEstimated, 1.0, parentTripCount.get())); return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()])); }
/** * Checks whether the parent breaker has been tripped */ public void checkParentLimit(String label) throws CircuitBreakingException { long totalUsed = 0; for (CircuitBreaker breaker : this.breakers.values()) { totalUsed += (breaker.getUsed() * breaker.getOverhead()); } long parentLimit = this.parentSettings.getLimit(); if (totalUsed > parentLimit) { this.parentTripCount.incrementAndGet(); final StringBuilder message = new StringBuilder("[parent] Data too large, data for [" + label + "]" + " would be [" + totalUsed + "/" + new ByteSizeValue(totalUsed) + "]" + ", which is larger than the limit of [" + parentLimit + "/" + new ByteSizeValue(parentLimit) + "]"); message.append(", usages ["); message.append(String.join(", ", this.breakers.entrySet().stream().map(e -> { final CircuitBreaker breaker = e.getValue(); final long breakerUsed = (long)(breaker.getUsed() * breaker.getOverhead()); return e.getKey() + "=" + breakerUsed + "/" + new ByteSizeValue(breakerUsed); }) .collect(Collectors.toList()))); message.append("]"); throw new CircuitBreakingException(message.toString(), totalUsed, parentLimit); } }
iterator = new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName); } else { breaker.addEstimateBytesAndMaybeBreak(estimatedBytes, fieldName);
@Test public void testCircuitBreakerAdjustmentOnIntTermsSet() { HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService( Settings.builder().build(), new NodeSettingsService(Settings.EMPTY)); CircuitBreaker breaker = hcbs.getBreaker(CircuitBreaker.REQUEST); assertThat(breaker.getUsed(), is(equalTo(0L))); IntegerTermsSet termsSet = new IntegerTermsSet(8, hcbs.getBreaker(CircuitBreaker.REQUEST)); long usedMem = breaker.getUsed(); assertThat(usedMem, greaterThan(0L)); for (int i = 0; i < 16; i++) { termsSet.add(i); } assertThat(breaker.getUsed(), greaterThan(usedMem)); termsSet.release(); assertThat(breaker.getUsed(), is(equalTo(0L))); }
@Override public CircuitBreakerStats stats(String name) { CircuitBreaker breaker = this.breakers.get(name); return new CircuitBreakerStats(breaker.getName(), breaker.getLimit(), breaker.getUsed(), breaker.getOverhead(), breaker.getTrippedCount()); }
/** * Increment or decrement the number of bytes that have been allocated to service * this request and potentially trigger a {@link CircuitBreakingException}. The * number of bytes allocated is automatically decremented with the circuit breaker * service on closure of this aggregator. * If memory has been returned, decrement it without tripping the breaker. * For performance reasons subclasses should not call this millions of times * each with small increments and instead batch up into larger allocations. * * @param bytes the number of bytes to register or negative to deregister the bytes * @return the cumulative size in bytes allocated by this aggregator to service this request */ protected long addRequestCircuitBreakerBytes(long bytes) { // Only use the potential to circuit break if bytes are being incremented if (bytes > 0) { this.breakerService .getBreaker(CircuitBreaker.REQUEST) .addEstimateBytesAndMaybeBreak(bytes, "<agg [" + name + "]>"); } else { this.breakerService .getBreaker(CircuitBreaker.REQUEST) .addWithoutBreaking(bytes); } this.requestBytesUsed += bytes; return requestBytesUsed; } /**
private void release(boolean isExceptionResponse) { if (released.compareAndSet(false, true)) { assert (releaseBy = new Exception()) != null; // easier to debug if it's already closed transport.getInFlightRequestBreaker().addWithoutBreaking(-reservedBytes); } else if (isExceptionResponse == false) { // only fail if we are not sending an error - we might send the error triggered by the previous // sendResponse call throw new IllegalStateException("reserved bytes are already released", releaseBy); } }
@Override public AllCircuitBreakerStats stats() { long parentEstimated = 0; List<CircuitBreakerStats> allStats = new ArrayList<>(this.breakers.size()); // Gather the "estimated" count for the parent breaker by adding the // estimations for each individual breaker for (CircuitBreaker breaker : this.breakers.values()) { allStats.add(stats(breaker.getName())); parentEstimated += breaker.getUsed(); } // Manually add the parent breaker settings since they aren't part of the breaker map allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(), parentEstimated, 1.0, parentTripCount.get())); return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()])); }
/** * Checks whether the parent breaker has been tripped */ public void checkParentLimit(String label) throws CircuitBreakingException { long totalUsed = 0; for (CircuitBreaker breaker : this.breakers.values()) { totalUsed += (breaker.getUsed() * breaker.getOverhead()); } long parentLimit = this.parentSettings.getLimit(); if (totalUsed > parentLimit) { this.parentTripCount.incrementAndGet(); throw new CircuitBreakingException("[parent] Data too large, data for [" + label + "] would be larger than limit of [" + parentLimit + "/" + new ByteSizeValue(parentLimit) + "]", totalUsed, parentLimit); } }
/** * Flush the {@code flushBuffer} to the breaker, incrementing the total * bytes and resetting the buffer. */ public void flush() { breaker.addEstimateBytesAndMaybeBreak(this.flushBuffer, this.fieldName); this.totalBytes += this.flushBuffer; this.flushBuffer = 0; }
@Test public void testCircuitBreakerAdjustmentOnLongTermsSet() { HierarchyCircuitBreakerService hcbs = new HierarchyCircuitBreakerService( Settings.builder().build(), new NodeSettingsService(Settings.EMPTY)); CircuitBreaker breaker = hcbs.getBreaker(CircuitBreaker.REQUEST); assertThat(breaker.getUsed(), is(equalTo(0L))); LongTermsSet termsSet = new LongTermsSet(8, hcbs.getBreaker(CircuitBreaker.REQUEST)); long usedMem = breaker.getUsed(); assertThat(usedMem, greaterThan(0L)); for (int i = 0; i < 16; i++) { termsSet.add(i); } assertThat(breaker.getUsed(), greaterThan(usedMem)); termsSet.release(); assertThat(breaker.getUsed(), is(equalTo(0L))); }
@Override public CircuitBreakerStats stats(String name) { CircuitBreaker breaker = this.breakers.get(name); return new CircuitBreakerStats(breaker.getName(), breaker.getLimit(), breaker.getUsed(), breaker.getOverhead(), breaker.getTrippedCount()); }