public void start() { streamIds.forEach(streamId -> { sendCallbackLatency.put(streamId, new SamzaHistogram(metricsRegistry, streamId, SEND_CALLBACK_LATENCY)); sendLatency.put(streamId, new SamzaHistogram(metricsRegistry, streamId, SEND_LATENCY)); sendErrors.put(streamId, metricsRegistry.newCounter(streamId, SEND_ERRORS)); }); if (aggSendLatency == null) { aggSendLatency = new SamzaHistogram(metricsRegistry, AGGREGATE, SEND_LATENCY); aggSendCallbackLatency = new SamzaHistogram(metricsRegistry, AGGREGATE, SEND_CALLBACK_LATENCY); aggSendErrors = metricsRegistry.newCounter(AGGREGATE, SEND_ERRORS); } }
/** * Updates the MetricsRegistery of this operator * @param startProcessing = begin processing of the message * @param endProcessing = end of processing */ private void updateMetrics(Instant startProcessing, Instant endProcessing) { queryInputEvents.inc(); processingTime.update(Duration.between(startProcessing, endProcessing).toMillis()); }
public void visit(MetricsVisitor visitor) { updateGaugeValues(percentile); visitor.gauge(this); } }
/** * update metrics given a message * @param message the input message * @return the same message */ @Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message) { Instant endProcessing = Instant.now(); Instant beginProcessing = Instant.parse(message.getSamzaSqlRelMsgMetadata().operatorBeginProcessingInstant); outputEvents.inc(); processingTime.update(Duration.between(beginProcessing, endProcessing).toMillis()); return message; } } // TranslatorOutputMetricsMapFunction
@Override public void init(Context context) { TranslatorContext translatorContext = ((SamzaSqlApplicationContext) context.getApplicationTaskContext()).getTranslatorContexts().get(queryId); this.samzaMsgConverter = translatorContext.getMsgConverter(outputTopic); ContainerContext containerContext = context.getContainerContext(); metricsRegistry = containerContext.getContainerMetricsRegistry(); /* insert (SendToOutputStream) metrics */ insertProcessingTime = new SamzaHistogram(metricsRegistry, insertLogicalId, TranslatorConstants.TOTAL_LATENCY_NAME);; /* query metrics */ totalLatency = new SamzaHistogram(metricsRegistry, queryLogicalId, TranslatorConstants.TOTAL_LATENCY_NAME);; queryLatency = new SamzaHistogram(metricsRegistry, queryLogicalId, TranslatorConstants.QUERY_LATENCY_NAME); queueingLatency = new SamzaHistogram(metricsRegistry, queryLogicalId, TranslatorConstants.QUEUEING_LATENCY_NAME);; queryOutputEvents = metricsRegistry.newCounter(queryLogicalId, TranslatorConstants.OUTPUT_EVENTS_NAME); queryOutputEvents.clear(); }
/** * Updates the Diagnostics Metrics (processing time and number of events) * @param arrivalTime input message arrival time (= beging of processing in this operator) * @param outputTime output message output time (=end of processing in this operator) * @param isNewInputMessage whether the input Message is from new input message or not */ private void updateMetrics(Instant arrivalTime, Instant outputTime, boolean isNewInputMessage) { if (isNewInputMessage) { inputEvents.inc(); } outputEvents.inc(); processingTime.update(Duration.between(arrivalTime, outputTime).toMillis()); }
.collect(Collectors.toMap(Function.identity(), x -> registry.newCounter(x, EVENT_BYTE_READ_RATE))); consumptionLagMs = streamIds.stream() .collect(Collectors.toMap(Function.identity(), x -> new SamzaHistogram(registry, x, CONSUMPTION_LAG_MS))); readErrors = streamIds.stream().collect(Collectors.toMap(Function.identity(), x -> registry.newCounter(x, READ_ERRORS))); aggEventReadRate = registry.newCounter(AGGREGATE, EVENT_READ_RATE); aggEventByteReadRate = registry.newCounter(AGGREGATE, EVENT_BYTE_READ_RATE); aggConsumptionLagMs = new SamzaHistogram(registry, AGGREGATE, CONSUMPTION_LAG_MS); aggReadErrors = registry.newCounter(AGGREGATE, READ_ERRORS);
/** * {@inheritDoc} */ @Override public synchronized void send(String source, OutgoingMessageEnvelope envelope) { checkForSendCallbackErrors("Received exception on message send"); String streamName = envelope.getSystemStream().getStream(); String streamId = physicalToStreamIds.getOrDefault(streamName, streamName); long beforeSendTimeMs = System.currentTimeMillis(); CompletableFuture<Void> sendResult = sendAsync(source, envelope); long afterSendTimeMs = System.currentTimeMillis(); long latencyMs = afterSendTimeMs - beforeSendTimeMs; sendLatency.get(streamId).update(latencyMs); aggSendLatency.update(latencyMs); pendingFutures.add(sendResult); // Auto update the metrics and possible throwable when futures are complete. sendResult.handle((aVoid, throwable) -> { long callbackLatencyMs = System.currentTimeMillis() - afterSendTimeMs; sendCallbackLatency.get(streamId).update(callbackLatencyMs); aggSendCallbackLatency.update(callbackLatencyMs); if (throwable != null) { sendErrors.get(streamId).inc(); aggSendErrors.inc(); LOG.error("Send message to event hub: {} failed with exception: ", streamId, throwable); sendExceptionOnCallback.compareAndSet(null, throwable); } return aVoid; }); }
/** * initializes the TranslatorOutputMetricsMapFunction before any message is processed * @param context the {@link Context} for this task */ @Override public void init(Context context) { ContainerContext containerContext = context.getContainerContext(); metricsRegistry = containerContext.getContainerMetricsRegistry(); processingTime = new SamzaHistogram(metricsRegistry, logicalOpId, TranslatorConstants.PROCESSING_TIME_NAME); outputEvents = metricsRegistry.newCounter(logicalOpId, TranslatorConstants.OUTPUT_EVENTS_NAME); outputEvents.clear(); }
/** * Updates the MetricsRegistery of this operator * @param startProcessing = begin processing of the message * @param endProcessing = end of processing */ private void updateMetrics(Instant startProcessing, boolean isOutput, Instant endProcessing) { inputEvents.inc(); if (isOutput) { outputEvents.inc(); } else { filteredOutEvents.inc(); } processingTime.update(Duration.between(startProcessing, endProcessing).toMillis()); }
@Override public void init(Context context) { TranslatorContext translatorContext = ((SamzaSqlApplicationContext) context.getApplicationTaskContext()).getTranslatorContexts().get(queryId); this.msgConverter = translatorContext.getMsgConverter(streamName); ContainerContext containerContext = context.getContainerContext(); metricsRegistry = containerContext.getContainerMetricsRegistry(); processingTime = new SamzaHistogram(metricsRegistry, logicalOpId, TranslatorConstants.PROCESSING_TIME_NAME); queryInputEvents = metricsRegistry.newCounter(queryLogicalId, TranslatorConstants.INPUT_EVENTS_NAME); queryInputEvents.clear(); }
/** * Updates the Diagnostics Metrics (processing time and number of events) * @param beginProcessing when sendOutput Started processing this message * @param endProcessing when sendOutput finished processing this message * @param metadata the event's message metadata */ private void updateMetrics(Instant beginProcessing, Instant endProcessing, SamzaSqlRelMsgMetadata metadata) { /* insert (SendToOutputStream) metrics */ insertProcessingTime.update(Duration.between(beginProcessing, endProcessing).toMillis()); /* query metrics */ Instant outputTime = Instant.now(); queryOutputEvents.inc(); /* TODO: remove scanTime validation once code to assign it is stable */ Validate.isTrue(metadata.hasScanTime()); Instant scanTime = Instant.parse(metadata.getscanTime()); queryLatency.update(Duration.between(scanTime, outputTime).toMillis()); /** TODO: change if hasArrivalTime to validation once arrivalTime is assigned, and later remove the check once code is stable */ if (metadata.hasArrivalTime()) { Instant arrivalTime = Instant.parse(metadata.getarrivalTime()); queueingLatency.update(Duration.between(arrivalTime, scanTime).toMillis()); } /* since availability of eventTime depends on source, we need the following check */ if (metadata.hasEventTime()) { Instant eventTime = Instant.parse(metadata.getEventTime()); totalLatency.update(Duration.between(eventTime, outputTime).toMillis()); } } } // OutputMapFunction
@Override public void init(Context context) { this.translatorContext = ((SamzaSqlApplicationContext) context.getApplicationTaskContext()).getTranslatorContexts().get(queryId); this.filter = (LogicalFilter) this.translatorContext.getRelNode(filterId); this.expr = this.translatorContext.getExpressionCompiler().compile(filter.getInputs(), Collections.singletonList(filter.getCondition())); ContainerContext containerContext = context.getContainerContext(); metricsRegistry = containerContext.getContainerMetricsRegistry(); processingTime = new SamzaHistogram(metricsRegistry, logicalOpId, TranslatorConstants.PROCESSING_TIME_NAME); inputEvents = metricsRegistry.newCounter(logicalOpId, TranslatorConstants.INPUT_EVENTS_NAME); inputEvents.clear(); filteredOutEvents = metricsRegistry.newCounter(logicalOpId, TranslatorConstants.FILTERED_EVENTS_NAME); filteredOutEvents.clear(); outputEvents = metricsRegistry.newCounter(logicalOpId, TranslatorConstants.OUTPUT_EVENTS_NAME); outputEvents.clear(); }
private void updateMetrics(EventData event) { int eventDataLength = event.getBytes() == null ? 0 : event.getBytes().length; eventReadRate.inc(); aggEventReadRate.inc(); eventByteReadRate.inc(eventDataLength); aggEventByteReadRate.inc(eventDataLength); long latencyMs = Duration.between(event.getSystemProperties().getEnqueuedTime(), Instant.now()).toMillis(); readLatency.update(latencyMs); aggConsumptionLagMs.update(latencyMs); }
/** * initializes the ProjectMapFunction before any message is processed * @param context the {@link Context} for this task */ @Override public void init(Context context) { this.translatorContext = ((SamzaSqlApplicationContext) context.getApplicationTaskContext()).getTranslatorContexts().get(queryId); this.project = (Project) this.translatorContext.getRelNode(projectId); this.expr = this.translatorContext.getExpressionCompiler().compile(project.getInputs(), project.getProjects()); ContainerContext containerContext = context.getContainerContext(); metricsRegistry = containerContext.getContainerMetricsRegistry(); processingTime = new SamzaHistogram(metricsRegistry, logicalOpId, TranslatorConstants.PROCESSING_TIME_NAME); inputEvents = metricsRegistry.newCounter(logicalOpId, TranslatorConstants.INPUT_EVENTS_NAME); inputEvents.clear(); outputEvents = metricsRegistry.newCounter(logicalOpId, TranslatorConstants.OUTPUT_EVENTS_NAME); outputEvents.clear(); }