@Override public void checkQuota(int numWrites, int numReads, int numScans) throws RpcThrottlingException { writeConsumed = estimateConsume(OperationType.MUTATE, numWrites, 100); readConsumed = estimateConsume(OperationType.GET, numReads, 100); readConsumed += estimateConsume(OperationType.SCAN, numScans, 1000); writeCapacityUnitConsumed = calculateWriteCapacityUnit(writeConsumed); readCapacityUnitConsumed = calculateReadCapacityUnit(readConsumed); writeAvailable = Long.MAX_VALUE; readAvailable = Long.MAX_VALUE; for (final QuotaLimiter limiter : limiters) { if (limiter.isBypass()) continue; limiter.checkQuota(numWrites, writeConsumed, numReads + numScans, readConsumed, writeCapacityUnitConsumed, readCapacityUnitConsumed); readAvailable = Math.min(readAvailable, limiter.getReadAvailable()); writeAvailable = Math.min(writeAvailable, limiter.getWriteAvailable()); } for (final QuotaLimiter limiter : limiters) { limiter.grabQuota(numWrites, writeConsumed, numReads + numScans, readConsumed, writeCapacityUnitConsumed, readCapacityUnitConsumed); } }
@Override public void close() { // Adjust the quota consumed for the specified operation long writeDiff = operationSize[OperationType.MUTATE.ordinal()] - writeConsumed; long readDiff = operationSize[OperationType.GET.ordinal()] + operationSize[OperationType.SCAN.ordinal()] - readConsumed; long writeCapacityUnitDiff = calculateWriteCapacityUnitDiff( operationSize[OperationType.MUTATE.ordinal()], writeConsumed); long readCapacityUnitDiff = calculateReadCapacityUnitDiff( operationSize[OperationType.GET.ordinal()] + operationSize[OperationType.SCAN.ordinal()], readConsumed); for (final QuotaLimiter limiter : limiters) { if (writeDiff != 0) { limiter.consumeWrite(writeDiff, writeCapacityUnitDiff); } if (readDiff != 0) { limiter.consumeRead(readDiff, readCapacityUnitDiff); } } }
private void assertNoThrottleException(final QuotaLimiter limiter, final int availReqs) { for (int i = 0; i < availReqs; ++i) { try { limiter.checkQuota(1, 1, 0, 0, 1, 0); } catch (RpcThrottlingException e) { fail("Unexpected RpcThrottlingException after " + i + " requests. limit=" + availReqs); } limiter.grabQuota(1, 1, 0, 0, 1, 0); } }
private <K> Map<K, QuotaLimiter> setLimiter(Map<K, QuotaLimiter> limiters, final K key, final Quotas quotas) { if (limiters == null) { limiters = new HashMap<>(); } QuotaLimiter limiter = quotas.hasThrottle() ? QuotaLimiterFactory.fromThrottle(quotas.getThrottle()) : null; if (limiter != null && !limiter.isBypass()) { limiters.put(key, limiter); } else { limiters.remove(key); } return limiters; }
private void assertThrottleException(final QuotaLimiter limiter, final int availReqs) { assertNoThrottleException(limiter, availReqs); try { limiter.checkQuota(1, 1, 0, 0, 1, 0); fail("Should have thrown RpcThrottlingException"); } catch (RpcThrottlingException e) { // expected } }
@Override public void close() { // Calculate and set the average size of get, scan and mutate for the current operation long getSize = avgOpSize.getAvgOperationSize(OperationType.GET); long scanSize = avgOpSize.getAvgOperationSize(OperationType.SCAN); long mutationSize = avgOpSize.getAvgOperationSize(OperationType.MUTATE); for (final QuotaLimiter limiter : limiters) { limiter.addOperationSize(OperationType.GET, getSize); limiter.addOperationSize(OperationType.SCAN, scanSize); limiter.addOperationSize(OperationType.MUTATE, mutationSize); } // Adjust the quota consumed for the specified operation long writeDiff = avgOpSize.getOperationSize(OperationType.MUTATE) - writeConsumed; long readDiff = (avgOpSize.getOperationSize(OperationType.GET) + avgOpSize .getOperationSize(OperationType.SCAN)) - readConsumed; for (final QuotaLimiter limiter : limiters) { if (writeDiff != 0) limiter.consumeWrite(writeDiff); if (readDiff != 0) limiter.consumeRead(readDiff); } }
private long estimateConsume(final OperationType type, int numReqs, long avgSize) { if (numReqs > 0) { for (final QuotaLimiter limiter : limiters) { long size = limiter.getAvgOperationSize(type); if (size > 0) { avgSize = size; break; } } return avgSize * numReqs; } return 0; } }
private void assertNoThrottleException(final QuotaLimiter limiter, final int availReqs) { for (int i = 0; i < availReqs; ++i) { try { limiter.checkQuota(1, 1, 0, 0); } catch (RpcThrottlingException e) { fail("Unexpected RpcThrottlingException after " + i + " requests. limit=" + availReqs); } limiter.grabQuota(1, 1, 0, 0); } }
UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi); QuotaLimiter userLimiter = userQuotaState.getTableLimiter(table); boolean useNoop = userLimiter.isBypass(); if (userQuotaState.hasBypassGlobals()) { if (LOG.isTraceEnabled()) { QuotaLimiter rsLimiter = quotaCache .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY); useNoop &= tableLimiter.isBypass() && nsLimiter.isBypass() && rsLimiter.isBypass(); if (LOG.isTraceEnabled()) { LOG.trace("get quota for ugi=" + ugi + " table=" + table + " userLimiter=" + userLimiter
@Test(timeout = 60000) public void testTableThrottleWithBatch() { final TableName TABLE_A = TableName.valueOf("TableA"); final int TABLE_A_THROTTLE_1 = 3; final long LAST_UPDATE_1 = 10; UserQuotaState quotaInfo = new UserQuotaState(); assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add A table limiters UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); QuotaLimiter limiter = quotaInfo.getTableLimiter(TABLE_A); try { limiter.checkQuota(TABLE_A_THROTTLE_1 + 1, TABLE_A_THROTTLE_1 + 1, 0, 0, 1, 0); fail("Should have thrown RpcThrottlingException"); } catch (RpcThrottlingException e) { // expected } }
@Override public void checkQuota(int numWrites, int numReads, int numScans) throws ThrottlingException { writeConsumed = estimateConsume(OperationType.MUTATE, numWrites, 100); readConsumed = estimateConsume(OperationType.GET, numReads, 100); readConsumed += estimateConsume(OperationType.SCAN, numScans, 1000); writeAvailable = Long.MAX_VALUE; readAvailable = Long.MAX_VALUE; for (final QuotaLimiter limiter : limiters) { if (limiter.isBypass()) continue; limiter.checkQuota(writeConsumed, readConsumed); readAvailable = Math.min(readAvailable, limiter.getReadAvailable()); writeAvailable = Math.min(writeAvailable, limiter.getWriteAvailable()); } for (final QuotaLimiter limiter : limiters) { limiter.grabQuota(writeConsumed, readConsumed); } }
private void assertThrottleException(final QuotaLimiter limiter, final int availReqs) { assertNoThrottleException(limiter, availReqs); try { limiter.checkQuota(1, 1, 0, 0); fail("Should have thrown RpcThrottlingException"); } catch (RpcThrottlingException e) { // expected } }
private <K> Map<K, QuotaLimiter> setLimiter(Map<K, QuotaLimiter> limiters, final K key, final Quotas quotas) { if (limiters == null) { limiters = new HashMap<K, QuotaLimiter>(); } QuotaLimiter limiter = quotas.hasThrottle() ? QuotaLimiterFactory.fromThrottle(quotas.getThrottle()) : null; if (limiter != null && !limiter.isBypass()) { limiters.put(key, limiter); } else { limiters.remove(key); } return limiters; }
@Test(timeout = 60000) public void testTableThrottleWithBatch() { final TableName TABLE_A = TableName.valueOf("TableA"); final int TABLE_A_THROTTLE_1 = 3; final long LAST_UPDATE_1 = 10; UserQuotaState quotaInfo = new UserQuotaState(); assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add A table limiters UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); QuotaLimiter limiter = quotaInfo.getTableLimiter(TABLE_A); try { limiter.checkQuota(TABLE_A_THROTTLE_1 + 1, TABLE_A_THROTTLE_1 + 1, 0, 0); fail("Should have thrown RpcThrottlingException"); } catch (RpcThrottlingException e) { // expected } }
UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi); QuotaLimiter userLimiter = userQuotaState.getTableLimiter(table); boolean useNoop = userLimiter.isBypass(); if (userQuotaState.hasBypassGlobals()) { if (LOG.isTraceEnabled()) { QuotaLimiter nsLimiter = quotaCache.getNamespaceLimiter(table.getNamespaceAsString()); QuotaLimiter tableLimiter = quotaCache.getTableLimiter(table); useNoop &= tableLimiter.isBypass() && nsLimiter.isBypass(); if (LOG.isTraceEnabled()) { LOG.trace("get quota for ugi=" + ugi + " table=" + table + " userLimiter=" + userLimiter