static Limiter createLimiter(Configuration configuration, SharedResourcesBroker<SimpleScopeType> broker) { try { Limiter limiter = new NoopLimiter(); long localQps = configuration.getLong(LOCALLY_ENFORCED_QPS, 0); if (localQps > 0) { log.info("Setting up local qps " + localQps); limiter = new MultiLimiter(limiter, new RateBasedLimiter(localQps)); } if (configuration.getBoolean(USE_THROTTLING_SERVER, false)) { log.info("Setting up remote throttling."); String resourceId = configuration.get(RESOURCE_ID); Limiter globalLimiter = broker.getSharedResource(new RestliLimiterFactory<SimpleScopeType>(), new SharedLimiterKey(resourceId)); limiter = new MultiLimiter(limiter, globalLimiter); } return limiter; } catch (NotConfiguredException nce) { throw new RuntimeException(nce); } }
public MultiLimiter(Limiter... underlyingLimiters) { ImmutableList.Builder<Limiter> builder = ImmutableList.builder(); Set<Limiter> seenLimiters = Sets.newHashSet(); for (Limiter limiter : underlyingLimiters) { if (limiter instanceof MultiLimiter) { for (Limiter innerLimiter : ((MultiLimiter) limiter).underlyingLimiters) { addLimiterIfNotSeen(innerLimiter, builder, seenLimiters); } } else if (!(limiter instanceof NoopLimiter)) { addLimiterIfNotSeen(limiter, builder, seenLimiters); } } this.underlyingLimiters = builder.build(); }
@Test public void test() throws Exception { CountBasedLimiter countLimiter1 = new CountBasedLimiter(3); CountBasedLimiter countLimiter2 = new CountBasedLimiter(1); MultiLimiter multiLimiter = new MultiLimiter(countLimiter1, countLimiter2); // Can only take 1 permit (limiter2 has only 1 permit available) Assert.assertNotNull(multiLimiter.acquirePermits(1)); Assert.assertNull(multiLimiter.acquirePermits(1)); // limiter1 has 1 leftover permit (one consumed in the failed second permit above) Assert.assertNotNull(countLimiter1.acquirePermits(1)); Assert.assertNull(countLimiter1.acquirePermits(1)); // limiter2 has not leftover permits Assert.assertNull(countLimiter2.acquirePermits(1)); }
public void testConstructor() throws Exception { CountBasedLimiter countLimiter1 = new CountBasedLimiter(3); CountBasedLimiter countLimiter2 = new CountBasedLimiter(3); CountBasedLimiter countLimiter3 = new CountBasedLimiter(3); NoopLimiter noopLimiter = new NoopLimiter(); MultiLimiter multiLimiter1 = new MultiLimiter(countLimiter1, countLimiter2); Assert.assertEquals(multiLimiter1.getUnderlyingLimiters(), Lists.newArrayList(countLimiter1, countLimiter2)); // Noop limiters get filtered MultiLimiter multiLimiter2 = new MultiLimiter(countLimiter1, noopLimiter); Assert.assertEquals(multiLimiter2.getUnderlyingLimiters(), Lists.newArrayList(countLimiter1)); // multilimiters get expanded MultiLimiter multiLimiter3 = new MultiLimiter(multiLimiter1, countLimiter3); Assert.assertEquals(multiLimiter3.getUnderlyingLimiters(), Lists.newArrayList(countLimiter1, countLimiter2, countLimiter3)); // deduplication MultiLimiter multiLimiter4 = new MultiLimiter(countLimiter1, countLimiter1); Assert.assertEquals(multiLimiter4.getUnderlyingLimiters(), Lists.newArrayList(countLimiter1)); // deduplication on expanded multilimiters MultiLimiter multiLimiter5 = new MultiLimiter(multiLimiter1, countLimiter1); Assert.assertEquals(multiLimiter5.getUnderlyingLimiters(), Lists.newArrayList(countLimiter1, countLimiter2)); }
@Test public void testMultiLevelLimiter() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, CountBasedLimiter.Factory.COUNT_KEY), "10", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, CountBasedLimiter.Factory.COUNT_KEY), "5" )); SharedResourcesBroker<SimpleScopeType> localBroker1 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local1")).build(); SharedResourcesBroker<SimpleScopeType> localBroker2 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local2")).build(); SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>(); Limiter limiter1 = ((ResourceInstance<Limiter>) factory.createResource(localBroker1, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Limiter limiter2 = ((ResourceInstance<Limiter>) factory.createResource(localBroker2, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Assert.assertTrue(limiter1 instanceof MultiLimiter); Assert.assertTrue(limiter2 instanceof MultiLimiter); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter1).getUnderlyingLimiters().get(0)).getCountLimit(), 5); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter1).getUnderlyingLimiters().get(1)).getCountLimit(), 10); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)).getCountLimit(), 5); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)).getCountLimit(), 10); Assert.assertNotEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(0), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)); Assert.assertEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(1), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)); }
/** * Throttles an {@link InputStream} if throttling is configured. * @param inputStream {@link InputStream} to throttle. * @param sourceURI used for selecting the throttling policy. * @param targetURI used for selecting the throttling policy. */ @Builder(builderMethodName = "throttleInputStream", builderClassName = "InputStreamThrottler") private ThrottledInputStream doThrottleInputStream(InputStream inputStream, URI sourceURI, URI targetURI) { Preconditions.checkNotNull(inputStream, "InputStream cannot be null."); Limiter limiter = new NoopLimiter(); if (sourceURI != null && targetURI != null) { StreamCopierSharedLimiterKey key = new StreamCopierSharedLimiterKey(sourceURI, targetURI); try { limiter = new MultiLimiter(limiter, this.broker.getSharedResource(new SharedLimiterFactory<S>(), key)); } catch (NotConfiguredException nce) { log.warn("Could not create a Limiter for key " + key, nce); } } else { log.info("Not throttling input stream because source or target URIs are not defined."); } Optional<MeteredInputStream> meteredStream = MeteredInputStream.findWrappedMeteredInputStream(inputStream); if (!meteredStream.isPresent()) { meteredStream = Optional.of(MeteredInputStream.builder().in(inputStream).build()); inputStream = meteredStream.get(); } return new ThrottledInputStream(inputStream, limiter, meteredStream.get()); }
public MultiLimiter(Limiter... underlyingLimiters) { ImmutableList.Builder<Limiter> builder = ImmutableList.builder(); Set<Limiter> seenLimiters = Sets.newHashSet(); for (Limiter limiter : underlyingLimiters) { if (limiter instanceof MultiLimiter) { for (Limiter innerLimiter : ((MultiLimiter) limiter).underlyingLimiters) { addLimiterIfNotSeen(innerLimiter, builder, seenLimiters); } } else if (!(limiter instanceof NoopLimiter)) { addLimiterIfNotSeen(limiter, builder, seenLimiters); } } this.underlyingLimiters = builder.build(); }
try { for (S parentScope : parentScopes) { limiter = new MultiLimiter(limiter, broker.getSharedResourceAtScope(this, configView.getKey(), parentScope));
/** * Throttles an {@link InputStream} if throttling is configured. * @param inputStream {@link InputStream} to throttle. * @param sourceURI used for selecting the throttling policy. * @param targetURI used for selecting the throttling policy. */ @Builder(builderMethodName = "throttleInputStream", builderClassName = "InputStreamThrottler") private ThrottledInputStream doThrottleInputStream(InputStream inputStream, URI sourceURI, URI targetURI) { Preconditions.checkNotNull(inputStream, "InputStream cannot be null."); Limiter limiter = new NoopLimiter(); if (sourceURI != null && targetURI != null) { StreamCopierSharedLimiterKey key = new StreamCopierSharedLimiterKey(sourceURI, targetURI); try { limiter = new MultiLimiter(limiter, this.broker.getSharedResource(new SharedLimiterFactory<S>(), key)); } catch (NotConfiguredException nce) { log.warn("Could not create a Limiter for key " + key, nce); } } else { log.info("Not throttling input stream because source or target URIs are not defined."); } Optional<MeteredInputStream> meteredStream = MeteredInputStream.findWrappedMeteredInputStream(inputStream); if (!meteredStream.isPresent()) { meteredStream = Optional.of(MeteredInputStream.builder().in(inputStream).build()); inputStream = meteredStream.get(); } return new ThrottledInputStream(inputStream, limiter, meteredStream.get()); }
try { for (S parentScope : parentScopes) { limiter = new MultiLimiter(limiter, broker.getSharedResourceAtScope(this, configView.getKey(), parentScope));