/** * @param event an event that was pulled off the queue * * @return a key to use for batching events together, by default this uses the sender of the * event, but sub-classes should override this to batch by something else */ protected String getBatchKey(final E event) { return event.getSender(); }
public ChannelHandler(final SelectionKey key, final D dispatcher, final Charset charset, final EventFactory<E> eventFactory, final BlockingQueue<E> events, final ComponentLog logger) { this.key = key; this.dispatcher = dispatcher; this.charset = charset; this.eventFactory = eventFactory; this.logger = logger; this.events = new EventQueue<E>(events, logger); }
@Override public StandardEvent create(final byte[] data, final Map<String, String> metadata, final ChannelResponder responder) { String sender = null; if (metadata != null && metadata.containsKey(EventFactory.SENDER_KEY)) { sender = metadata.get(EventFactory.SENDER_KEY); } return new StandardEvent(sender, data, responder); }
public void handle(final BeatsFrame frame, final ChannelResponder<SocketChannel> responder, final String sender) throws IOException, InterruptedException { final Map<String, String> metadata = EventFactoryUtil.createMapWithSender(sender.toString()); metadata.put(BeatsMetadata.SEQNUMBER_KEY, String.valueOf(frame.getSeqNumber())); String line = ""; /* If frameType is a JSON , parse the frame payload into a JsonElement so that all JSON elements but "message" are inserted into the event metadata. As per above, the "message" element gets added into the body of the event */ if (frame.getFrameType() == FRAME_JSON ) { // queue the raw event blocking until space is available, reset the buffer final E event = eventFactory.create(frame.getPayload(), metadata, responder); events.offer(event); } } }
@Override protected String getTransitUri(FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender; final String transitUri = new StringBuilder().append("udp").append("://").append(senderHost).append(":") .append(port).toString(); return transitUri; }
@Override protected ChannelDispatcher createDispatcher(final ProcessContext context, final BlockingQueue<StandardEvent> events) throws IOException { final String sendingHost = context.getProperty(SENDING_HOST).evaluateAttributeExpressions().getValue(); final Integer sendingHostPort = context.getProperty(SENDING_HOST_PORT).evaluateAttributeExpressions().asInteger(); final Integer bufferSize = context.getProperty(RECV_BUFFER_SIZE).asDataSize(DataUnit.B).intValue(); final BlockingQueue<ByteBuffer> bufferPool = createBufferPool(context.getMaxConcurrentTasks(), bufferSize); final EventFactory<StandardEvent> eventFactory = new StandardEventFactory(); return new DatagramChannelDispatcher<>(eventFactory, bufferPool, events, getLogger(), sendingHost, sendingHostPort); }
/** * Process the contents of the buffer. Give sub-classes a chance to override this behavior. * * @param sslSocketChannel the channel the data was read from * @param socketChannel the socket channel being wrapped by sslSocketChannel * @param bytesRead the number of bytes read * @param buffer the buffer to process * @throws InterruptedException thrown if interrupted while queuing events */ protected void processBuffer(final SSLSocketChannel sslSocketChannel, final SocketChannel socketChannel, final int bytesRead, final byte[] buffer) throws InterruptedException, IOException { final InetAddress sender = socketChannel.socket().getInetAddress(); // go through the buffer looking for the end of each message for (int i = 0; i < bytesRead; i++) { final byte currByte = buffer[i]; // check if at end of a message if (currByte == getDelimiter()) { if (currBytes.size() > 0) { final SSLSocketChannelResponder response = new SSLSocketChannelResponder(socketChannel, sslSocketChannel); final Map<String, String> metadata = EventFactoryUtil.createMapWithSender(sender.toString()); final E event = eventFactory.create(currBytes.toByteArray(), metadata, response); events.offer(event); currBytes.reset(); } } else { currBytes.write(currByte); } } }
public BeatsFrameHandler(final SelectionKey selectionKey, final Charset charset, final EventFactory<E> eventFactory, final BlockingQueue<E> events, final AsyncChannelDispatcher dispatcher, final ComponentLog logger) { this.key = selectionKey; this.charset = charset; this.eventFactory = eventFactory; this.dispatcher = dispatcher; this.logger = logger; this.events = new EventQueue<>(events, logger); this.encoder = new BeatsEncoder(); }
@Test public void testSuccessWithBatchSizeGreaterThanAvailableRecords() { final String sender = "foo"; final StandardEvent event1 = new StandardEvent(sender, DATAGRAM_1.getBytes(StandardCharsets.UTF_8), null); proc.addEvent(event1); final StandardEvent event2 = new StandardEvent(sender, DATAGRAM_2.getBytes(StandardCharsets.UTF_8), null); proc.addEvent(event2); final StandardEvent event3 = new StandardEvent(sender, DATAGRAM_3.getBytes(StandardCharsets.UTF_8), null); proc.addEvent(event3); runner.run(); runner.assertAllFlowFilesTransferred(ListenUDPRecord.REL_SUCCESS, 1); final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenUDPRecord.REL_SUCCESS).get(0); flowFile.assertAttributeEquals(ListenUDPRecord.RECORD_COUNT_ATTR, "3"); }
@Override protected String getTransitUri(FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender; final String transitUri = new StringBuilder().append("tcp").append("://").append(senderHost).append(":") .append(port).toString(); return transitUri; }
@Override protected ChannelDispatcher createDispatcher(final ProcessContext context, final BlockingQueue<StandardEvent> events) throws IOException { final String sendingHost = context.getProperty(SENDING_HOST).evaluateAttributeExpressions().getValue(); final Integer sendingHostPort = context.getProperty(SENDING_HOST_PORT).evaluateAttributeExpressions().asInteger(); final Integer bufferSize = context.getProperty(RECV_BUFFER_SIZE).asDataSize(DataUnit.B).intValue(); final BlockingQueue<ByteBuffer> bufferPool = createBufferPool(context.getMaxConcurrentTasks(), bufferSize); final EventFactory<StandardEvent> eventFactory = new StandardEventFactory(); return new DatagramChannelDispatcher<>(eventFactory, bufferPool, events, getLogger(), sendingHost, sendingHostPort); }
if (currBytes.size() > 0) { final SocketChannelResponder response = new SocketChannelResponder(socketChannel); final Map<String, String> metadata = EventFactoryUtil.createMapWithSender(sender.toString()); final E event = eventFactory.create(currBytes.toByteArray(), metadata, response); events.offer(event); currBytes.reset();
public DatagramChannelDispatcher(final EventFactory<E> eventFactory, final BlockingQueue<ByteBuffer> bufferPool, final BlockingQueue<E> events, final ComponentLog logger, final String sendingHost, final Integer sendingPort) { this.eventFactory = eventFactory; this.bufferPool = bufferPool; this.logger = logger; this.sendingHost = sendingHost; this.sendingPort = sendingPort; this.events = new EventQueue<>(events, logger); if (bufferPool == null || bufferPool.size() == 0) { throw new IllegalArgumentException("A pool of available ByteBuffers is required"); } }
@Test public void testMultipleRecordsPerDatagram() { final String sender = "foo"; final StandardEvent event1 = new StandardEvent(sender, MULTI_DATAGRAM_1.getBytes(StandardCharsets.UTF_8), null); proc.addEvent(event1); final StandardEvent event2 = new StandardEvent(sender, MULTI_DATAGRAM_2.getBytes(StandardCharsets.UTF_8), null); proc.addEvent(event2); runner.run(); runner.assertAllFlowFilesTransferred(ListenUDPRecord.REL_SUCCESS, 1); final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenUDPRecord.REL_SUCCESS).get(0); flowFile.assertAttributeEquals(ListenUDPRecord.RECORD_COUNT_ATTR, "6"); }
@Override protected Map<String, String> getAttributes(final FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final Map<String,String> attributes = new HashMap<>(3); attributes.put("tcp.sender", sender); attributes.put("tcp.port", String.valueOf(port)); return attributes; }
public void handle(final LumberjackFrame frame, final ChannelResponder<SocketChannel> responder, final String sender) throws IOException, InterruptedException { final Map<String, String> metadata = EventFactoryUtil.createMapWithSender(sender.toString()); metadata.put(LumberjackMetadata.SEQNUMBER_KEY, String.valueOf(frame.getSeqNumber())); String line = ""; final E event = eventFactory.create(line.getBytes(), metadata, responder); events.offer(event); } else if (frame.getFrameType() == 0x4A ) { logger.error("Data type was JSON. JSON payload aren't yet supported, pending the documentation of Lumberjack protocol v2");
public RELPFrameHandler(final SelectionKey selectionKey, final Charset charset, final EventFactory<E> eventFactory, final BlockingQueue<E> events, final AsyncChannelDispatcher dispatcher, final ComponentLog logger) { this.key = selectionKey; this.charset = charset; this.eventFactory = eventFactory; this.dispatcher = dispatcher; this.logger = logger; this.events = new EventQueue<>(events, logger); this.encoder = new RELPEncoder(charset); }
@Override protected Map<String, String> getAttributes(final FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final Map<String,String> attributes = new HashMap<>(3); attributes.put(UDP_SENDER_ATTR, sender); attributes.put(UDP_PORT_ATTR, String.valueOf(port)); return attributes; }
public void handle(final RELPFrame frame, final ChannelResponder<SocketChannel> responder, final String sender) throws IOException, InterruptedException { // respond to open and close commands immediately, create and queue an event for everything else if (CMD_OPEN.equals(frame.getCommand())) { Map<String,String> offers = RELPResponse.parseOffers(frame.getData(), charset); ChannelResponse response = new RELPChannelResponse(encoder, RELPResponse.open(frame.getTxnr(), offers)); responder.addResponse(response); responder.respond(); } else if (CMD_CLOSE.equals(frame.getCommand())) { ChannelResponse response = new RELPChannelResponse(encoder, RELPResponse.ok(frame.getTxnr())); responder.addResponse(response); responder.respond(); dispatcher.completeConnection(key); } else { final Map<String, String> metadata = EventFactoryUtil.createMapWithSender(sender.toString()); metadata.put(RELPMetadata.TXNR_KEY, String.valueOf(frame.getTxnr())); metadata.put(RELPMetadata.COMMAND_KEY, frame.getCommand()); final E event = eventFactory.create(frame.getData(), metadata, responder); events.offer(event); } }
public LumberjackFrameHandler(final SelectionKey selectionKey, final Charset charset, final EventFactory<E> eventFactory, final BlockingQueue<E> events, final AsyncChannelDispatcher dispatcher, final ComponentLog logger) { this.key = selectionKey; this.charset = charset; this.eventFactory = eventFactory; this.dispatcher = dispatcher; this.logger = logger; this.events = new EventQueue<>(events, logger); this.encoder = new LumberjackEncoder(); }