Refine search
public void onStatus(Status status) { Record doc = extractRecord("", avroSchema, status); if (doc == null) { return; // skip } docs.add(doc); if (docs.size() >= maxBatchSize || System.currentTimeMillis() >= batchEndTime) { batchEndTime = System.currentTimeMillis() + maxBatchDurationMillis; byte[] bytes; try { bytes = serializeToAvro(avroSchema, docs); } catch (IOException e) { LOGGER.error("Exception while serializing tweet", e); return; //skip } Event event = EventBuilder.withBody(bytes); getChannelProcessor().processEvent(event); // send event to the flume sink docs.clear(); } docCount++; if ((docCount % REPORT_INTERVAL) == 0) { LOGGER.info(String.format("Processed %s docs", numFormatter.format(docCount))); } if ((docCount % STATS_INTERVAL) == 0) { logStats(); } }
public ResultCode Log(List<LogEntry> list) throws TException { if (list != null) { sourceCounter.addToEventReceivedCount(list.size()); try { List<Event> events = new ArrayList<Event>(list.size()); for (LogEntry entry : list) { Map<String, String> headers = new HashMap<String, String>(1, 1); String category = entry.getCategory(); if (category != null) { headers.put(SCRIBE_CATEGORY, category); } Event event = EventBuilder.withBody(entry.getMessage().getBytes(), headers); events.add(event); } if (events.size() > 0) { getChannelProcessor().processEventBatch(events); } sourceCounter.addToEventAcceptedCount(list.size()); return ResultCode.OK; } catch (Exception e) { LOG.warn("Scribe source handling failure", e); sourceCounter.incrementEventReadOrChannelFail(e); } } return ResultCode.TRY_LATER; } }
@Override public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException { if (source instanceof PollableSource) { super.onTrigger(context, sessionFactory); } else if (source instanceof EventDrivenSource) { ProcessSessionFactory old = sessionFactoryRef.getAndSet(sessionFactory); if (old != sessionFactory) { if (runnerRef.get() != null) { stopped(); sessionFactoryRef.set(sessionFactory); } runnerRef.set(new EventDrivenSourceRunner()); eventDrivenSourceChannelRef.set(new NifiSessionFactoryChannel(sessionFactoryRef.get(), SUCCESS)); eventDrivenSourceChannelRef.get().start(); source.setChannelProcessor(new ChannelProcessor( new NifiChannelSelector(eventDrivenSourceChannelRef.get()))); runnerRef.get().setSource(source); runnerRef.get().start(); } } }
if (batchSize == 1) { if (eventsSentTX < totalEvents) { getChannelProcessor().processEvent( EventBuilder.withBody(String.valueOf(eventsSentTX++).getBytes())); sourceCounter.incrementEventAcceptedCount(); } else { for (int i = 0; i < batchSize; i++) { if (eventsSentTX < totalEvents) { batchArrayList.add(i, EventBuilder.withBody(String .valueOf(eventsSentTX++).getBytes())); } else { getChannelProcessor().processEventBatch(batchArrayList); sourceCounter.incrementAppendBatchAcceptedCount(); sourceCounter.addToEventAcceptedCount(batchArrayList.size());
limiter.acquire(); getChannelProcessor().processEvent(event); } else { long eventsLeft = maxTotalEvents - totalEventSent; getChannelProcessor().processEventBatch(eventBatchListToProcess);
public void put(Event event) throws ChannelException { getChannelProcessor().processEvent(event); }
public void putAll(List<Event> events) throws ChannelException { getChannelProcessor().processEventBatch(events); }
@Override public void stop() { runner.shouldStop.set(true); try { runnerThread.interrupt(); runnerThread.join(); } catch (InterruptedException e) { logger.warn("Interrupted while waiting for polling runner to stop. Please report this.", e); Thread.currentThread().interrupt(); } Source source = getSource(); source.stop(); ChannelProcessor cp = source.getChannelProcessor(); cp.close(); lifecycleState = LifecycleState.STOP; }
if (batchSize == 1) { if (eventsSentTX < totalEvents) { getChannelProcessor().processEvent( EventBuilder.withBody(String.valueOf(eventsSentTX++).getBytes())); sourceCounter.incrementEventAcceptedCount(); } else { for (int i = 0; i < batchSize; i++) { if (eventsSentTX < totalEvents) { batchArrayList.add(i, EventBuilder.withBody(String .valueOf(eventsSentTX++).getBytes())); } else { getChannelProcessor().processEventBatch(batchArrayList); sourceCounter.incrementAppendBatchAcceptedCount(); sourceCounter.addToEventAcceptedCount(batchArrayList.size());
@Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent mEvent) { try { Event e = extractEvent((ChannelBuffer)mEvent.getMessage(), mEvent.getRemoteAddress()); if (e == null) { return; } getChannelProcessor().processEvent(e); counterGroup.incrementAndGet("events.success"); } catch (ChannelException ex) { counterGroup.incrementAndGet("events.dropped"); logger.error("Error writing to channel", ex); } catch (RuntimeException ex) { counterGroup.incrementAndGet("events.dropped"); logger.error("Error retrieving event from udp stream, event dropped", ex); } } }
private void flushEventBatch(List<Event> eventList) { channelProcessor.processEventBatch(eventList); sourceCounter.addToEventAcceptedCount(eventList.size()); eventList.clear(); lastPushToChannel = systemClock.currentTimeMillis(); }
limiter.acquire(); getChannelProcessor().processEvent(event); } else { long eventsLeft = maxTotalEvents - totalEventSent; getChannelProcessor().processEventBatch(eventBatchListToProcess);
@Override public void stop() { Source source = getSource(); source.stop(); ChannelProcessor cp = source.getChannelProcessor(); cp.close(); lifecycleState = LifecycleState.STOP; }
public void append(ThriftFlumeEvent evt ) { if (evt == null) { return; } Map<String, String> headers = new HashMap<String, String>(); // extract Flume event headers headers.put(HOST, evt.getHost()); headers.put(TIMESTAMP, Long.toString(evt.getTimestamp())); headers.put(PRIORITY, evt.getPriority().toString()); headers.put(NANOS, Long.toString(evt.getNanos())); for (Entry<String, ByteBuffer> entry: evt.getFields().entrySet()) { headers.put(entry.getKey().toString(), UTF_8.decode(entry.getValue()).toString()); } headers.put(OG_EVENT, "yes"); Event event = EventBuilder.withBody(evt.getBody(), headers); counterGroup.incrementAndGet("rpc.events"); try { getChannelProcessor().processEvent(event); } catch (ChannelException ex) { LOG.warn("Failed to process event", ex); return; } counterGroup.incrementAndGet("rpc.successful"); return; }
@Override public Status appendBatch(List<ThriftFlumeEvent> events) throws TException { sourceCounter.incrementAppendBatchReceivedCount(); sourceCounter.addToEventReceivedCount(events.size()); List<Event> flumeEvents = Lists.newArrayList(); for (ThriftFlumeEvent event : events) { flumeEvents.add(EventBuilder.withBody(event.getBody(), event.getHeaders())); } try { getChannelProcessor().processEventBatch(flumeEvents); } catch (ChannelException ex) { logger.warn("Thrift source %s could not append events to the channel.", getName()); sourceCounter.incrementChannelWriteFail(); return Status.FAILED; } sourceCounter.incrementAppendBatchAcceptedCount(); sourceCounter.addToEventAcceptedCount(events.size()); return Status.OK; } }
@Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent mEvent) { try { syslogUtils.setEventSize(maxsize); Event e = syslogUtils.extractEvent((ChannelBuffer)mEvent.getMessage()); if (e == null) { return; } if (clientIPHeader != null) { e.getHeaders().put(clientIPHeader, SyslogUtils.getIP(mEvent.getRemoteAddress())); } if (clientHostnameHeader != null) { e.getHeaders().put(clientHostnameHeader, SyslogUtils.getHostname(mEvent.getRemoteAddress())); } sourceCounter.incrementEventReceivedCount(); getChannelProcessor().processEvent(e); sourceCounter.incrementEventAcceptedCount(); } catch (ChannelException ex) { logger.error("Error writting to channel", ex); sourceCounter.incrementChannelWriteFail(); return; } catch (RuntimeException ex) { logger.error("Error parsing event from syslog stream, event dropped", ex); sourceCounter.incrementEventReadFail(); return; } } }
sourceCounter.addToEventReceivedCount(events.size()); try { getChannelProcessor().processEventBatch(events); } catch (ChannelException ex) { LOG.warn("Error appending event to channel. "
sourceChannels, selectorConfig); ChannelProcessor channelProcessor = new ChannelProcessor(selector); Configurables.configure(channelProcessor, config); sourceChannels, selectorConfig); ChannelProcessor channelProcessor = new ChannelProcessor(selector); Configurables.configure(channelProcessor, context); source.setChannelProcessor(channelProcessor);
@Override public void stop() { runner.shouldStop.set(true); try { runnerThread.interrupt(); runnerThread.join(); } catch (InterruptedException e) { logger.warn("Interrupted while waiting for polling runner to stop. Please report this.", e); Thread.currentThread().interrupt(); } Source source = getSource(); source.stop(); ChannelProcessor cp = source.getChannelProcessor(); cp.close(); lifecycleState = LifecycleState.STOP; }
@Override public Void append( AvroFlumeOGEvent evt ) throws AvroRemoteException { counterGroup.incrementAndGet("rpc.received"); Map<String, String> headers = new HashMap<String, String>(); // extract Flume OG event headers headers.put(HOST, evt.getHost().toString()); headers.put(TIMESTAMP, evt.getTimestamp().toString()); headers.put(PRIORITY, evt.getPriority().toString()); headers.put(NANOS, evt.getNanos().toString()); for (Entry<CharSequence, ByteBuffer> entry : evt.getFields().entrySet()) { headers.put(entry.getKey().toString(), entry.getValue().toString()); } headers.put(OG_EVENT, "yes"); Event event = EventBuilder.withBody(evt.getBody().array(), headers); try { getChannelProcessor().processEvent(event); counterGroup.incrementAndGet("rpc.events"); } catch (ChannelException ex) { return null; } counterGroup.incrementAndGet("rpc.successful"); return null; }