private Worker runReaderInternal(List<String> serversetPaths, List<String> finagleNames, int truncationInterval) throws IOException { Preconditions.checkNotNull(dlUri); int ssid = null == startStreamId ? shardId * numStreams : startStreamId; int esid = null == endStreamId ? (shardId + readersPerStream) * numStreams : endStreamId; if (null != maxStreamId) { esid = Math.min(esid, maxStreamId); } return createReaderWorker( conf, dlUri, streamPrefix, ssid, esid, concurrency, serversetPaths, finagleNames, truncationInterval, readFromHead, statsReceiver, statsProvider.getStatsLogger("dlreader")); }
Worker runDLWriter() throws IOException { Preconditions.checkNotNull(dlUri, "dlUri must be defined"); Preconditions.checkArgument(rate > 0, "rate must be greater than 0"); Preconditions.checkArgument(maxRate >= rate, "max rate must be greater than rate"); Preconditions.checkArgument(changeRate >= 0, "change rate must be positive"); Preconditions.checkArgument(changeRateSeconds >= 0, "change rate must be positive"); Preconditions.checkArgument(concurrency > 0, "concurrency must be greater than 0"); ShiftableRateLimiter rateLimiter = new ShiftableRateLimiter(rate, maxRate, changeRate, changeRateSeconds, TimeUnit.SECONDS); return new DLWriterWorker(conf, dlUri, streamPrefix, shardId * numStreams, (shardId + 1) * numStreams, rateLimiter, concurrency, msgSize, statsProvider.getStatsLogger("dlwrite")); }
finagleNames, statsReceiver.scope("write_client"), statsProvider.getStatsLogger("write"), thriftmux, handshakeWithClientInfo,
protected void run(String[] args) throws Exception { logger.info("Parsing arguments for benchmark : {}", args); // parse command line parseCommandLine(args); statsProvider.start(conf); // run the benchmark StatsLogger statsLogger = statsProvider.getStatsLogger("dl"); DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() .conf(conf) .uri(uri) .statsLogger(statsLogger) .build(); try { benchmark(namespace, streamName, statsProvider.getStatsLogger("benchmark")); } finally { namespace.close(); statsProvider.stop(); } }
void runMonitor(DistributedLogConfiguration conf, URI dlUri) throws IOException { // stats statsProvider.getStatsLogger("monitor").registerGauge("num_streams", new org.apache.bookkeeper.stats.Gauge<Number>() { @Override public Number getDefaultValue() { return 0; } @Override public Number getSample() { return knownStreams.size(); } }); logger.info("Construct dl namespace @ {}", dlUri); dlNamespace = DistributedLogNamespaceBuilder.newBuilder() .conf(conf) .uri(dlUri) .build(); if (watchNamespaceChanges) { dlNamespace.registerNamespaceListener(this); } else { onStreamsChanged(dlNamespace.getLogs()); } }
StatsLogger perStreamStatsLogger; if (perStreamStatsEnabled) { perStreamStatsLogger = provider.getStatsLogger("stream"); } else { perStreamStatsLogger = NullStatsLogger.INSTANCE; dlUri, partitionConverter, provider.getStatsLogger(""), perStreamStatsLogger, keepAliveLatch); StatsLogger serviceStatsLogger = provider.getStatsLogger("service");
/** * clean up the gauge before we close to help GC. */ private void unregisterGauge(){ statsProvider.getStatsLogger("monitor").unregisterGauge("num_streams", numOfStreamsGauge); }
@Override public StatsLogger getStatsLogger(String scope) { StatsLogger statsLogger = statsLoggers.get(scope); if (null == statsLogger) { StatsLogger newStatsLogger = new CachingStatsLogger(underlying.getStatsLogger(scope)); StatsLogger oldStatsLogger = statsLoggers.putIfAbsent(scope, newStatsLogger); statsLogger = (null == oldStatsLogger) ? newStatsLogger : oldStatsLogger; } return statsLogger; }
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("benchmark-worker"); try { jc.parse(args); } catch (ParameterException e) { System.err.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } Configuration conf = new CompositeConfiguration(); conf.setProperty(Stats.STATS_PROVIDER_CLASS, PrometheusMetricsProvider.class.getName()); conf.setProperty("prometheusStatsHttpPort", arguments.statsPort); Stats.loadStatsProvider(conf); StatsProvider provider = Stats.get(); provider.start(conf); Runtime.getRuntime().addShutdownHook(new Thread( () -> provider.stop(), "benchmark-worker-shutdown-thread")); // Dump configuration variables log.info("Starting benchmark with config: {}", writer.writeValueAsString(arguments)); // Start web server Javalin app = Javalin.start(arguments.httpPort); new WorkerHandler(app, provider.getStatsLogger("benchmark")); }
void runMonitor(DistributedLogConfiguration conf, URI dlUri) throws IOException { // stats statsProvider.getStatsLogger("monitor").registerGauge("num_streams", numOfStreamsGauge); logger.info("Construct dl namespace @ {}", dlUri); dlNamespace = DistributedLogNamespaceBuilder.newBuilder() .conf(conf) .uri(dlUri) .build(); if (watchNamespaceChanges) { dlNamespace.registerNamespaceListener(this); } else { onStreamsChanged(dlNamespace.getLogs()); } }
StatsLogger rootStatsLogger = statsProviderService.getStatsProvider().getStatsLogger("");
checkNotNull(bookieConfig, "No ServerConfiguration for Bookie"); checkNotNull(bookieStatsProvider, "No Stats Provider for Bookie"); bookieServer = new BookieServer(bookieConfig, bookieStatsProvider.getStatsLogger("")); } else { bookieServer = null;
StatsLogger rootStatsLogger = statsProviderService.getStatsProvider().getStatsLogger("");
StatsLogger perStreamStatsLogger; if (perStreamStatsEnabled) { perStreamStatsLogger = provider.getStatsLogger("stream"); } else { perStreamStatsLogger = NullStatsLogger.INSTANCE; partitionConverter, routingService, provider.getStatsLogger(""), perStreamStatsLogger, keepAliveLatch, StatsLogger serviceStatsLogger = provider.getStatsLogger("service");