while (token == null && this.availableBytesLength != -1) { if (this.index >= this.availableBytesLength) { this.fill(); this.index = i + 1; int size = this.index - this.mark - this.delimiterBytes.length; token = this.extractDataToken(size); this.mark = this.index; j = 0; token = this.extractDataToken(this.index - this.mark);
/** * Constructs a new instance * * @param is * instance of {@link InputStream} representing the data * @param delimiterBytes * byte array representing delimiter bytes used to split the * input stream. Can be 'null'. NOTE: the 'null' is allowed only * for convenience and consistency since without delimiter this * class is no different then BufferedReader which reads the * entire stream into a byte array and there may be a more * efficient ways to do that (if that is the case). * @param maxDataSize * maximum size of data derived from the input stream. This means * that neither {@link InputStream} nor its individual tokens (if * delimiter is used) can ever be greater then this size. * @param initialBufferSize * initial size of the buffer used to buffer {@link InputStream} * or its parts (if delimiter is used) to create its byte[] * representation. Must be positive integer. The buffer will grow * automatically as needed up to the Integer.MAX_VALUE; * */ public StreamDemarcator(InputStream is, byte[] delimiterBytes, int maxDataSize, int initialBufferSize) { super(is, maxDataSize, initialBufferSize); this.validate(delimiterBytes); this.delimiterBytes = delimiterBytes; }
StreamDemarcator streamTokenizer = new StreamDemarcator(publishingContext.getContentStream(), publishingContext.getDelimiterBytes(), publishingContext.getMaxRequestSize()); boolean continueSending = true; KafkaPublisherResult result = null; for (; continueSending && (messageBytes = streamTokenizer.nextToken()) != null; tokenCounter++) { if (prevLastAckedMessageIndex < tokenCounter) { Integer partitionId = publishingContext.getPartitionId();
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
while (token == null && this.availableBytesLength != -1) { if (this.index >= this.availableBytesLength) { this.fill(); this.index = i + 1; int size = this.index - this.mark - this.delimiterBytes.length; token = this.extractDataToken(size); this.mark = this.index; j = 0; token = this.extractDataToken(this.index - this.mark);
/** * Constructs a new instance * * @param is * instance of {@link InputStream} representing the data * @param delimiterBytes * byte array representing delimiter bytes used to split the * input stream. Can be 'null'. NOTE: the 'null' is allowed only * for convenience and consistency since without delimiter this * class is no different then BufferedReader which reads the * entire stream into a byte array and there may be a more * efficient ways to do that (if that is the case). * @param maxDataSize * maximum size of data derived from the input stream. This means * that neither {@link InputStream} nor its individual tokens (if * delimiter is used) can ever be greater then this size. * @param initialBufferSize * initial size of the buffer used to buffer {@link InputStream} * or its parts (if delimiter is used) to create its byte[] * representation. Must be positive integer. The buffer will grow * automatically as needed up to the Integer.MAX_VALUE; * */ public StreamDemarcator(InputStream is, byte[] delimiterBytes, int maxDataSize, int initialBufferSize) { super(is, maxDataSize, initialBufferSize); this.validate(delimiterBytes); this.delimiterBytes = delimiterBytes; }
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
StreamDemarcator streamTokenizer = new StreamDemarcator(publishingContext.getContentStream(), publishingContext.getDelimiterBytes(), publishingContext.getMaxRequestSize()); boolean continueSending = true; KafkaPublisherResult result = null; for (; continueSending && (messageBytes = streamTokenizer.nextToken()) != null; tokenCounter++) { if (prevLastAckedMessageIndex < tokenCounter) { ProducerRecord<byte[], byte[]> message = new ProducerRecord<>(publishingContext.getTopic(), publishingContext.getKeyBytes(), messageBytes);
StreamDemarcator streamTokenizer = new StreamDemarcator(publishingContext.getContentStream(), publishingContext.getDelimiterBytes(), publishingContext.getMaxRequestSize()); boolean continueSending = true; KafkaPublisherResult result = null; for (; continueSending && (messageBytes = streamTokenizer.nextToken()) != null; tokenCounter++) { if (prevLastAckedMessageIndex < tokenCounter) { Integer partitionId = publishingContext.getPartitionId();
StreamDemarcator streamTokenizer = new StreamDemarcator(publishingContext.getContentStream(), publishingContext.getDelimiterBytes(), publishingContext.getMaxRequestSize()); boolean continueSending = true; KafkaPublisherResult result = null; for (; continueSending && (messageBytes = streamTokenizer.nextToken()) != null; tokenCounter++) { if (prevLastAckedMessageIndex < tokenCounter) { Integer partitionId = publishingContext.getPartitionId();
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);
try (final StreamDemarcator demarcator = new StreamDemarcator(flowFileContent, demarcatorBytes, maxMessageSize)) { while ((messageContent = demarcator.nextToken()) != null) { publish(flowFile, messageKey, messageContent, topic, tracker);