/** * Flush the {@code flushBuffer} to the breaker, incrementing the total * bytes and resetting the buffer. */ public void flush() { breaker.addEstimateBytesAndMaybeBreak(this.flushBuffer, this.fieldName); this.totalBytes += this.flushBuffer; this.flushBuffer = 0; }
default CircuitBreaker getInFlightRequestBreaker() { return new NoopCircuitBreaker("in-flight-noop"); }
@Override public CircuitBreakerStats stats(String name) { CircuitBreaker breaker = this.breakers.get(name); return new CircuitBreakerStats(breaker.getName(), breaker.getLimit(), breaker.getUsed(), breaker.getOverhead(), breaker.getTrippedCount()); }
circuitBreak(label, bytes); newUsed = noLimit(bytes, label); } else { newUsed = limit(bytes, label); this.addWithoutBreaking(-bytes); throw e;
/** * Checks whether the parent breaker has been tripped */ public void checkParentLimit(String label) throws CircuitBreakingException { long totalUsed = 0; for (CircuitBreaker breaker : this.breakers.values()) { totalUsed += (breaker.getUsed() * breaker.getOverhead()); } long parentLimit = this.parentSettings.getLimit(); if (totalUsed > parentLimit) { this.parentTripCount.incrementAndGet(); final StringBuilder message = new StringBuilder("[parent] Data too large, data for [" + label + "]" + " would be [" + totalUsed + "/" + new ByteSizeValue(totalUsed) + "]" + ", which is larger than the limit of [" + parentLimit + "/" + new ByteSizeValue(parentLimit) + "]"); message.append(", usages ["); message.append(String.join(", ", this.breakers.entrySet().stream().map(e -> { final CircuitBreaker breaker = e.getValue(); final long breakerUsed = (long)(breaker.getUsed() * breaker.getOverhead()); return e.getKey() + "=" + breakerUsed + "/" + new ByteSizeValue(breakerUsed); }) .collect(Collectors.toList()))); message.append("]"); throw new CircuitBreakingException(message.toString(), totalUsed, parentLimit); } }
breaker.addEstimateBytesAndMaybeBreak(delta, "<reused_arrays>"); } catch (CircuitBreakingException e) { if (isDataAlreadyCreated) { breaker.addWithoutBreaking(delta); breaker.addWithoutBreaking(delta); breaker.addWithoutBreaking(delta);
private void close() { // attempt to close once atomically if (closed.compareAndSet(false, true) == false) { throw new IllegalStateException("Channel is already closed"); } inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-contentLength); }
@Override public AllCircuitBreakerStats stats() { long parentEstimated = 0; List<CircuitBreakerStats> allStats = new ArrayList<>(this.breakers.size()); // Gather the "estimated" count for the parent breaker by adding the // estimations for each individual breaker for (CircuitBreaker breaker : this.breakers.values()) { allStats.add(stats(breaker.getName())); parentEstimated += breaker.getUsed(); } // Manually add the parent breaker settings since they aren't part of the breaker map allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(), parentEstimated, 1.0, parentTripCount.get())); return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()])); }
CircuitBreaker breaker = new NoopCircuitBreaker(breakerSettings.getName()); breakers.put(breakerSettings.getName(), breaker); } else { CircuitBreaker oldBreaker; CircuitBreaker breaker = new ChildMemoryCircuitBreaker(breakerSettings, LogManager.getLogger(CHILD_LOGGER_PREFIX + breakerSettings.getName()), this, breakerSettings.getName()); return; breaker = new ChildMemoryCircuitBreaker(breakerSettings, (ChildMemoryCircuitBreaker)oldBreaker, LogManager.getLogger(CHILD_LOGGER_PREFIX + breakerSettings.getName()),
/** * Method used to trip the breaker, delegates to the parent to determine * whether to trip the breaker or not */ @Override public void circuitBreak(String fieldName, long bytesNeeded) { this.trippedCount.incrementAndGet(); final String message = "[" + this.name + "] Data too large, data for [" + fieldName + "]" + " would be [" + bytesNeeded + "/" + new ByteSizeValue(bytesNeeded) + "]" + ", which is larger than the limit of [" + memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]"; logger.debug("{}", message); throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit); }
@Override public String toString() { return "[" + this.name + ",type=" + this.type.toString() + ",limit=" + this.limitBytes + "/" + new ByteSizeValue(this.limitBytes) + ",overhead=" + this.overhead + "]"; } }
private long limit(long bytes, String label) { long newUsed;// Otherwise, check the addition and commit the addition, looping if // there are conflicts. May result in additional logging, but it's // trace logging and shouldn't be counted on for additions. long currentUsed; do { currentUsed = this.used.get(); newUsed = currentUsed + bytes; long newUsedWithOverhead = (long) (newUsed * overheadConstant); if (logger.isTraceEnabled()) { logger.trace("[{}] Adding [{}][{}] to used bytes [new used: [{}], limit: {} [{}], estimate: {} [{}]]", this.name, new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed), memoryBytesLimit, new ByteSizeValue(memoryBytesLimit), newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead)); } if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) { logger.warn("[{}] New used memory {} [{}] for data of [{}] would be larger than configured breaker: {} [{}], breaking", this.name, newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label, memoryBytesLimit, new ByteSizeValue(memoryBytesLimit)); circuitBreak(label, newUsedWithOverhead); } // Attempt to set the new used value, but make sure it hasn't changed // underneath us, if it has, keep trying until we are able to set it } while (!this.used.compareAndSet(currentUsed, newUsed)); return newUsed; }
/** * Increment or decrement the number of bytes that have been allocated to service * this request and potentially trigger a {@link CircuitBreakingException}. The * number of bytes allocated is automatically decremented with the circuit breaker * service on closure of this aggregator. * If memory has been returned, decrement it without tripping the breaker. * For performance reasons subclasses should not call this millions of times * each with small increments and instead batch up into larger allocations. * * @param bytes the number of bytes to register or negative to deregister the bytes * @return the cumulative size in bytes allocated by this aggregator to service this request */ protected long addRequestCircuitBreakerBytes(long bytes) { // Only use the potential to circuit break if bytes are being incremented if (bytes > 0) { this.breakerService .getBreaker(CircuitBreaker.REQUEST) .addEstimateBytesAndMaybeBreak(bytes, "<agg [" + name + "]>"); } else { this.breakerService .getBreaker(CircuitBreaker.REQUEST) .addWithoutBreaking(bytes); } this.requestBytesUsed += bytes; return requestBytesUsed; } /**
@Override public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) { assert sizeInBytes >= 0 : "When reducing circuit breaker, it should be adjusted with a number higher or " + "equal to 0 and not [" + sizeInBytes + "]"; circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes); } });
iterator = new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName); } else { breaker.addEstimateBytesAndMaybeBreak(estimatedBytes, fieldName);
/** * Check whether there have been too many compilations within the last minute, throwing a circuit breaking exception if so. * This is a variant of the token bucket algorithm: https://en.wikipedia.org/wiki/Token_bucket * * It can be thought of as a bucket with water, every time the bucket is checked, water is added proportional to the amount of time that * elapsed since the last time it was checked. If there is enough water, some is removed and the request is allowed. If there is not * enough water the request is denied. Just like a normal bucket, if water is added that overflows the bucket, the extra water/capacity * is discarded - there can never be more water in the bucket than the size of the bucket. */ void checkCompilationLimit() { long now = System.nanoTime(); long timePassed = now - lastInlineCompileTime; lastInlineCompileTime = now; scriptsPerTimeWindow += (timePassed) * compilesAllowedPerNano; // It's been over the time limit anyway, readjust the bucket to be level if (scriptsPerTimeWindow > rate.v1()) { scriptsPerTimeWindow = rate.v1(); } // If there is enough tokens in the bucket, allow the request and decrease the tokens by 1 if (scriptsPerTimeWindow >= 1) { scriptsPerTimeWindow -= 1.0; } else { // Otherwise reject the request throw new CircuitBreakingException("[script] Too many dynamic script compilations within, max: [" + rate.v1() + "/" + rate.v2() +"]; please use indexed, or scripts with parameters instead; " + "this limit can be changed by the [" + SCRIPT_MAX_COMPILATIONS_RATE.getKey() + "] setting"); } }
/** * Adjust the circuit breaker now that terms have been loaded, getting * the actual used either from the parameter (if estimation worked for * the entire set), or from the TermsEnum if it has been wrapped in a * RamAccountingTermsEnum. * * @param termsEnum terms that were loaded * @param actualUsed actual field data memory usage */ @Override public void afterLoad(TermsEnum termsEnum, long actualUsed) { if (termsEnum instanceof RamAccountingTermsEnum) { estimatedBytes = ((RamAccountingTermsEnum) termsEnum).getTotalBytes(); } breaker.addWithoutBreaking(-(estimatedBytes - actualUsed)); }
private void release(boolean isExceptionResponse) { if (released.compareAndSet(false, true)) { assert (releaseBy = new Exception()) != null; // easier to debug if it's already closed transport.getInFlightRequestBreaker().addWithoutBreaking(-reservedBytes); } else if (isExceptionResponse == false) { // only fail if we are not sending an error - we might send the error triggered by the previous // sendResponse call throw new IllegalStateException("reserved bytes are already released", releaseBy); } }
/** Called upon release of the aggregator. */ @Override public void close() { try { doClose(); } finally { this.breakerService.getBreaker(CircuitBreaker.REQUEST).addWithoutBreaking(-this.requestBytesUsed); } }
/** * Build global ordinals for the provided {@link IndexReader}. */ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, IndexSettings indexSettings, CircuitBreakerService breakerService, Logger logger, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); final long memorySizeInBytes = ordinalMap.ramBytesUsed(); breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { logger.debug( "global-ordinals [{}][{}] took [{}]", indexFieldData.getFieldName(), ordinalMap.getValueCount(), new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS) ); } return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, memorySizeInBytes, scriptFunction ); }