public ProfileHBaseMapper(RowKeyBuilder rowKeyBuilder, ColumnBuilder columnBuilder) { setRowKeyBuilder(rowKeyBuilder); setColumnBuilder(columnBuilder); }
public FixedFrequencyFlushSignal(long flushFrequencyMillis) { if(flushFrequencyMillis < 0) { throw new IllegalArgumentException("flush frequency must be >= 0"); } this.flushFrequency = flushFrequencyMillis; reset(); }
public ProfileBuilderBolt withPeriodDuration(int duration, TimeUnit units) { return withPeriodDurationMillis(units.toMillis(duration)); }
@Override public void execute(TupleWindow window) { if(LOG.isDebugEnabled()) { log(window); } try { // handle each tuple in the window for(Tuple tuple : window.get()) { handleMessage(tuple); } // time to flush active profiles? if(activeFlushSignal.isTimeToFlush()) { flushActive(); } } catch (Throwable e) { LOG.error("Unexpected error", e); collector.reportError(e); } }
/** * Handles the processing of a single tuple. * * @param input The tuple containing a telemetry message. */ private void handleMessage(Tuple input) { // crack open the tuple JSONObject message = getField(MESSAGE_TUPLE_FIELD, input, JSONObject.class); ProfileConfig definition = getField(PROFILE_TUPLE_FIELD, input, ProfileConfig.class); String entity = getField(ENTITY_TUPLE_FIELD, input, String.class); Long timestamp = getField(TIMESTAMP_TUPLE_FIELD, input, Long.class); // keep track of time activeFlushSignal.update(timestamp); // distribute the message MessageRoute route = new MessageRoute(definition, entity, message, timestamp); synchronized (messageDistributor) { messageDistributor.distribute(route, getStellarContext()); } LOG.debug("Message distributed: profile={}, entity={}, timestamp={}", definition.getProfile(), entity, timestamp); }
this.messageDistributor = new DefaultMessageDistributor(periodDurationMillis, profileTimeToLiveMillis, maxNumberOfRoutes); this.configurations = new ProfilerConfigurations(); this.activeFlushSignal = new FixedFrequencyFlushSignal(periodDurationMillis); setupZookeeper(); startFlushingExpiredProfiles();
/** * Flush all active profiles. */ protected void flushActive() { activeFlushSignal.reset(); // flush the active profiles List<ProfileMeasurement> measurements; synchronized(messageDistributor) { measurements = messageDistributor.flush(); emitMeasurements(measurements); } LOG.debug("Flushed active profiles and found {} measurement(s).", measurements.size()); }
/** * Returns true, if it is time to flush. * * @return True if time to flush. Otherwise, false. */ @Override public boolean isTimeToFlush() { boolean flush = currentTime > flushTime; LOG.debug("Flush={}, '{}' ms until flush; currentTime={}, flushTime={}", flush, timeToNextFlush(), currentTime, flushTime); return flush; }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declareStream(getStreamId(), new Fields("measurement")); }
public ProfileBuilderBolt withProfileTimeToLive(int duration, TimeUnit units) { return withProfileTimeToLiveMillis(units.toMillis(duration)); }
/** * Creates a separate thread that regularly flushes expired profiles. */ private void startFlushingExpiredProfiles() { long initialDelay = profileTimeToLiveMillis; long period = profileTimeToLiveMillis; flushExpiredExecutor = Executors.newSingleThreadScheduledExecutor(); flushExpiredExecutor.scheduleAtFixedRate(() -> flushExpired(), initialDelay, period, TimeUnit.MILLISECONDS); }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { if(emitters.size() == 0) { throw new IllegalStateException("At least one destination handler must be defined."); } // allow each emitter to define its own stream emitters.forEach(emitter -> emitter.declareOutputFields(declarer)); }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { // the kafka writer expects a field named 'message' declarer.declareStream(getStreamId(), new Fields("message")); }
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { super.prepare(stormConf, context, collector); this.collector = collector; this.parser = new JSONParser(); this.router = new DefaultMessageRouter(getStellarContext()); }
/** * Flushes all expired profiles. * * <p>If a profile has not received a message for an extended period of time then it is * marked as expired. Periodically we need to flush these expired profiles to ensure * that their state is not lost. */ protected void flushExpired() { List<ProfileMeasurement> measurements = null; try { // flush the expired profiles synchronized (messageDistributor) { measurements = messageDistributor.flushExpired(); emitMeasurements(measurements); } } catch(Throwable t) { // need to catch the exception, otherwise subsequent executions would be suppressed. // see java.util.concurrent.ScheduledExecutorService#scheduleAtFixedRate LOG.error("Failed to flush expired profiles", t); } LOG.debug("Flushed expired profiles and found {} measurement(s).", CollectionUtils.size(measurements)); }
public Context getStellarContext() { Map<String, Object> global = getConfigurations().getGlobalConfig(); return new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .with(Context.Capabilities.GLOBAL_CONFIG, () -> global) .with(Context.Capabilities.STELLAR_CONFIG, () -> global) .build(); }
private Context getStellarContext() { Map<String, Object> global = getConfigurations().getGlobalConfig(); return new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> zookeeperClient) .with(Context.Capabilities.GLOBAL_CONFIG, () -> global) .with(Context.Capabilities.STELLAR_CONFIG, () -> global) .build(); }
public ProfileHBaseMapper() { setRowKeyBuilder(new SaltyRowKeyBuilder()); setColumnBuilder(new ValueOnlyColumnBuilder()); }
/** * Update the internal state which tracks time. * * @param timestamp The timestamp received within a tuple. */ @Override public void update(long timestamp) { if(timestamp > currentTime) { // need to update current time LOG.debug("Updating current time; last={}, new={}", currentTime, timestamp); currentTime = timestamp; } else if ((currentTime - timestamp) > flushFrequency) { // significantly out-of-order timestamps LOG.warn("Timestamps out-of-order by '{}' ms. This may indicate a problem in the data. last={}, current={}", (currentTime - timestamp), timestamp, currentTime); } if(flushTime == 0) { // set the next time to flush flushTime = currentTime + flushFrequency; LOG.debug("Setting flush time; '{}' ms until flush; flushTime={}, currentTime={}, flushFreq={}", timeToNextFlush(), flushTime, currentTime, flushFrequency); } }
@Override public void emit(ProfileMeasurement measurement, OutputCollector collector) { // measurements are always emitted to hbase collector.emit(getStreamId(), new Values(measurement)); LOG.debug("Emitted measurement; stream={}, profile={}, entity={}, period={}, start={}, end={}", getStreamId(), measurement.getProfileName(), measurement.getEntity(), measurement.getPeriod().getPeriod(), measurement.getPeriod().getStartTimeMillis(), measurement.getPeriod().getEndTimeMillis()); }