protected void run(String[] args) throws Exception { logger.info("Parsing arguments for benchmark : {}", args); // parse command line parseCommandLine(args); statsProvider.start(conf); // run the benchmark StatsLogger statsLogger = statsProvider.getStatsLogger("dl"); DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() .conf(conf) .uri(uri) .statsLogger(statsLogger) .build(); try { benchmark(namespace, streamName, statsProvider.getStatsLogger("benchmark")); } finally { namespace.close(); statsProvider.stop(); } }
private Worker runReaderInternal(List<String> serversetPaths, List<String> finagleNames, int truncationInterval) throws IOException { Preconditions.checkNotNull(dlUri); int ssid = null == startStreamId ? shardId * numStreams : startStreamId; int esid = null == endStreamId ? (shardId + readersPerStream) * numStreams : endStreamId; if (null != maxStreamId) { esid = Math.min(esid, maxStreamId); } return createReaderWorker( conf, dlUri, streamPrefix, ssid, esid, concurrency, serversetPaths, finagleNames, truncationInterval, readFromHead, statsReceiver, statsProvider.getStatsLogger("dlreader")); }
@Override public void run() { logger.info("Closing DistributedLog Server."); server.close(); logger.info("Closed DistributedLog Server."); statsProvider.stop(); } });
statsProvider.start(dlConf);
@Override public String getStatsName(String... statsComponents) { return underlying.getStatsName(statsComponents); } }
statsProvider.writeAllMetrics(writer); writer.flush(); response.setCode(StatusCode.OK);
Worker runDLWriter() throws IOException { Preconditions.checkNotNull(dlUri, "dlUri must be defined"); Preconditions.checkArgument(rate > 0, "rate must be greater than 0"); Preconditions.checkArgument(maxRate >= rate, "max rate must be greater than rate"); Preconditions.checkArgument(changeRate >= 0, "change rate must be positive"); Preconditions.checkArgument(changeRateSeconds >= 0, "change rate must be positive"); Preconditions.checkArgument(concurrency > 0, "concurrency must be greater than 0"); ShiftableRateLimiter rateLimiter = new ShiftableRateLimiter(rate, maxRate, changeRate, changeRateSeconds, TimeUnit.SECONDS); return new DLWriterWorker(conf, dlUri, streamPrefix, shardId * numStreams, (shardId + 1) * numStreams, rateLimiter, concurrency, msgSize, statsProvider.getStatsLogger("dlwrite")); }
/** * Close the server */ public void close() { logger.info("Closing monitor service."); if (null != dlClient) { dlClient.close(); } if (null != zkServerSets) { for (DLZkServerSet zkServerSet : zkServerSets) { zkServerSet.close(); } } if (null != dlNamespace) { dlNamespace.close(); } executorService.shutdown(); try { if (!executorService.awaitTermination(1, TimeUnit.MINUTES)) { executorService.shutdownNow(); } } catch (InterruptedException e) { logger.error("Interrupted on waiting shutting down monitor executor service : ", e); } if (null != statsProvider) { statsProvider.stop(); } keepAliveLatch.countDown(); logger.info("Closed monitor service."); }
statsProvider.start(dlConf); String[] serverSetPaths = StringUtils.split(serverSetArg.get(), ","); if (serverSetPaths.length == 0) {
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("benchmark-worker"); try { jc.parse(args); } catch (ParameterException e) { System.err.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } Configuration conf = new CompositeConfiguration(); conf.setProperty(Stats.STATS_PROVIDER_CLASS, PrometheusMetricsProvider.class.getName()); conf.setProperty("prometheusStatsHttpPort", arguments.statsPort); Stats.loadStatsProvider(conf); StatsProvider provider = Stats.get(); provider.start(conf); Runtime.getRuntime().addShutdownHook(new Thread( () -> provider.stop(), "benchmark-worker-shutdown-thread")); // Dump configuration variables log.info("Starting benchmark with config: {}", writer.writeValueAsString(arguments)); // Start web server Javalin app = Javalin.start(arguments.httpPort); new WorkerHandler(app, provider.getStatsLogger("benchmark")); }
finagleNames, statsReceiver.scope("write_client"), statsProvider.getStatsLogger("write"), thriftmux, handshakeWithClientInfo,
/** * Close the server. */ public void close() { if (null != announcer) { try { announcer.unannounce(); } catch (IOException e) { logger.warn("Error on unannouncing service : ", e); } announcer.close(); } closeServer(Pair.of(dlService, server), gracefulShutdownMs, TimeUnit.MILLISECONDS); if (null != statsProvider) { statsProvider.stop(); } SchedulerUtils.shutdownScheduler(configExecutorService, 60, TimeUnit.SECONDS); keepAliveLatch.countDown(); }
@Override protected void doStart() { statsProvider.start(conf); }
void runMonitor(DistributedLogConfiguration conf, URI dlUri) throws IOException { // stats statsProvider.getStatsLogger("monitor").registerGauge("num_streams", new org.apache.bookkeeper.stats.Gauge<Number>() { @Override public Number getDefaultValue() { return 0; } @Override public Number getSample() { return knownStreams.size(); } }); logger.info("Construct dl namespace @ {}", dlUri); dlNamespace = DistributedLogNamespaceBuilder.newBuilder() .conf(conf) .uri(dlUri) .build(); if (watchNamespaceChanges) { dlNamespace.registerNamespaceListener(this); } else { onStreamsChanged(dlNamespace.getLogs()); } }
server.close(); logger.info("Closed DistributedLog Server."); statsProvider.stop();
@Override public void start(Configuration conf) { this.underlying.start(conf); }
StatsLogger perStreamStatsLogger; if (perStreamStatsEnabled) { perStreamStatsLogger = provider.getStatsLogger("stream"); } else { perStreamStatsLogger = NullStatsLogger.INSTANCE; dlUri, partitionConverter, provider.getStatsLogger(""), perStreamStatsLogger, keepAliveLatch); StatsLogger serviceStatsLogger = provider.getStatsLogger("service");
@Override protected void doStop() { statsProvider.stop(); }
public void start() throws Exception { if (bookieStatsProvider != null) { bookieStatsProvider.start(bookieConfig); log.info("started bookieStatsProvider."); } if (bookieServer != null) { bookieServer.start(); log.info("started bookieServer."); } if (autoRecoveryMain != null) { autoRecoveryMain.start(); log.info("started bookie autoRecoveryMain."); } pulsarService.start(); log.info("PulsarService started."); }