@Override public void doRun(Limiter limiter) throws InterruptedException { int ops = 0; while (ops < this.opsTarget) { limiter.acquirePermits(1); ops++; } } }
@Override public Closeable acquirePermits(long permits) throws InterruptedException { Closeable closeable = this.underlying.acquirePermits(permits); this.localPermitCount.addAndGet(permits); return closeable; }
@Override public void doRun(Limiter limiter) throws InterruptedException { long runForSeconds = new Random().nextInt(180) + 1; long endTime = System.currentTimeMillis() + runForSeconds * 1000; while (System.currentTimeMillis() < endTime) { limiter.acquirePermits(1); } } }
@Override public Closeable acquirePermits(long permits) throws InterruptedException { Closer closer = Closer.create(); for (Limiter limiter : this.underlyingLimiters) { Closeable permit = limiter.acquirePermits(permits); if (permit == null) { try { closer.close(); } catch (IOException ioe) { throw new RuntimeException("Could not return intermediate permits."); } return null; } closer.register(permit); } return closer; }
/** * Acquire permit along with emitting metrics if enabled. * @param permits * @throws InterruptedException */ private void acquirePermits(long permits) throws InterruptedException { long startMs = System.currentTimeMillis(); //Measure in milliseconds. (Nanoseconds are more precise but expensive and not worth for this case) limiter.acquirePermits(permits); long permitAcquisitionTime = System.currentTimeMillis() - startMs; if (throttledTimer.isPresent()) { // Metrics enabled Instrumented.updateTimer(throttledTimer, permitAcquisitionTime, TimeUnit.MILLISECONDS); } this.throttledTime += permitAcquisitionTime; }
public final RP sendRequest(RQ request) throws IOException { final Timer.Context context = sendTimer.time(); try { if (limiter.acquirePermits(1) != null) { log.debug ("Acquired permits successfully"); return sendRequestImpl (request); } else { throw new IOException ("Acquired permits return null"); } } catch (InterruptedException e) { throw new IOException("Throttling is interrupted"); } finally { context.stop(); } }
public final void sendAsyncRequest(RQ request, Callback<RP> callback) throws IOException { final Timer.Context context = sendTimer.time(); try { if (limiter.acquirePermits(1) != null) { log.debug ("Acquired permits successfully"); sendAsyncRequestImpl (request, callback); } else { throw new IOException ("Acquired permits return null"); } } catch (InterruptedException e) { throw new IOException("Throttling is interrupted"); } finally { context.stop(); } }
@Override public D readRecord(@Deprecated D reuse) throws DataRecordException, IOException { try (Closer closer = Closer.create()) { if (closer.register(this.limiter.acquirePermits(1)) != null) { return this.extractor.readRecord(reuse); } submitLimiterStopMetadataEvents(); return null; } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Interrupted while trying to acquire the next permit", ie); } }
private void blockUntilPermitsAvailable() { try { long currentCount = this.meter.getBytesProcessedMeter().getCount(); long permitsNeeded = currentCount - this.prevCount; this.prevCount = currentCount; Closeable permit = this.limiter.acquirePermits(permitsNeeded); if (permit == null) { throw new RuntimeException("Could not acquire permits."); } } catch (InterruptedException ie) { throw new RuntimeException(ie); } } }
@Test public void testThrottling() throws InterruptedException { Assert.assertTrue(this.limiter.acquirePermits(1) != null); Thread.sleep(1000); Assert.assertTrue(this.limiter.acquirePermits(1) != null); Thread.sleep(1000); Assert.assertTrue(this.limiter.acquirePermits(1) != null); Thread.sleep(1100); Assert.assertTrue(this.limiter.acquirePermits(1) == null); }
private void acquirePermits(int permits, String op) throws IOException { try { Closeable closeable = getRateLimiter().acquirePermits(permits); if (closeable == null) { throw new NotEnoughPermitsException(op); } } catch (InterruptedException e) { throw new NotEnoughPermitsException(op, e); } }
@Test public void testThrottling() throws InterruptedException { Limiter limiter = new CountBasedLimiter(10); limiter.start(); for (int i = 0; i < 10; i++) { Assert.assertTrue(limiter.acquirePermits(1) != null); } Assert.assertTrue(limiter.acquirePermits(1) == null); limiter.stop(); } }
@Test public void testThrottling() throws InterruptedException { Meter meter = new Meter(); for (int i = 0; i < 100; i++) { Assert.assertTrue(this.limiter.acquirePermits(1) != null); meter.mark(); Thread.sleep((RANDOM.nextInt() & Integer.MAX_VALUE) % 10); } // Assert a fuzzy equal with 5% of tolerance Assert.assertTrue(DoubleMath.fuzzyEquals(meter.getMeanRate(), 20d, 20d * 0.05)); }
Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555")); Mockito.when(taskState.getTaskId()).thenReturn("888"); Mockito.when(limiter.acquirePermits(1)).thenReturn(null); Mockito.when (taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN)).thenReturn("file://xyz"); Mockito.when(workUnit.getProperties()).thenReturn(properties);
Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555")); Mockito.when(taskState.getTaskId()).thenReturn("888"); Mockito.when(limiter.acquirePermits(1)).thenReturn(null); Mockito.when (taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN)).thenReturn("file://xyz"); Mockito.when(workUnit.getProperties()).thenReturn(properties);
Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555")); Mockito.when(taskState.getTaskId()).thenReturn("888"); Mockito.when(limiter.acquirePermits(1)).thenReturn(null); Mockito.when (taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN)).thenReturn("file://xyz"); Mockito.when(workUnit.getProperties()).thenReturn(properties);
@Test public void testSkipGlobalLimiterOnLimiterFactory() throws Exception { Map<String, String> configMap = ImmutableMap.of( BrokerConfigurationKeyGenerator.generateKey(new SharedLimiterFactory(), null, null, SharedLimiterFactory.SKIP_GLOBAL_LIMITER_KEY), "true" ); SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.parseMap(configMap), SimpleScopeType.GLOBAL.defaultScopeInstance()); MyRequestSender requestSender = new MyRequestSender(); broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender); Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource")); Assert.assertNotNull(limiter.acquirePermits(10)); Assert.assertEquals(requestSender.requestList.size(), 0); broker.close(); }
@Test public void testRestliLimiterCalledByLimiterFactory() throws Exception { SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); MyRequestSender requestSender = new MyRequestSender(); broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender); Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource")); Assert.assertNotNull(limiter.acquirePermits(10)); Assert.assertEquals(requestSender.requestList.size(), 1); broker.close(); }
@Override public D readRecord(@Deprecated D reuse) throws DataRecordException, IOException { try (Closer closer = Closer.create()) { if (closer.register(this.limiter.acquirePermits(1)) != null) { return this.extractor.readRecord(reuse); } submitLimiterStopMetadataEvents(); return null; } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Interrupted while trying to acquire the next permit", ie); } }
private void acquirePermits(int permits, String op) throws IOException { try { Closeable closeable = getRateLimiter().acquirePermits(permits); if (closeable == null) { throw new NotEnoughPermitsException(op); } } catch (InterruptedException e) { throw new NotEnoughPermitsException(op, e); } }