public synchronized void addRecord(ConsumerRecord<K, V> record) { ensureNotClosed(); TopicPartition tp = new TopicPartition(record.topic(), record.partition()); Set<TopicPartition> currentAssigned = new HashSet<>(this.subscriptions.assignedPartitions()); if (!currentAssigned.contains(tp)) throw new IllegalStateException("Cannot add records for a partition that is not assigned to the consumer"); List<ConsumerRecord<K, V>> recs = this.records.computeIfAbsent(tp, k -> new ArrayList<>()); recs.add(record); }
@Nonnull @Override public List<OrderedPartitionableRecord<Integer, Long>> poll(long timeout) { List<OrderedPartitionableRecord<Integer, Long>> polledRecords = new ArrayList<>(); for (ConsumerRecord<byte[], byte[]> record : consumer.poll(timeout)) { polledRecords.add(new OrderedPartitionableRecord<>( record.topic(), record.partition(), record.offset(), record.value() == null ? null : ImmutableList.of(record.value()) )); } return polledRecords; }
public KafkaSpoutMessageId(ConsumerRecord<?, ?> consumerRecord, boolean nullTuple) { this(new TopicPartition(consumerRecord.topic(), consumerRecord.partition()), consumerRecord.offset(), nullTuple); }
void set(ConsumerRecord<byte[], byte[]> consumerRecord) { this.partition = consumerRecord.partition(); this.timestamp = consumerRecord.timestamp(); this.offset = consumerRecord.offset(); this.value = consumerRecord.value(); this.recordKey = consumerRecord.key(); }
@Override public List<Object> apply(ConsumerRecord<K, V> record) { return new Values(record.topic(), record.partition(), record.offset(), record.key(), record.value()); }
@Override public BaseConsumerRecord receive() { if (_recordIter == null || !_recordIter.hasNext()) _recordIter = _consumer.poll(Long.MAX_VALUE).iterator(); ConsumerRecord<String, String> record = _recordIter.next(); return new BaseConsumerRecord(record.topic(), record.partition(), record.offset(), record.key(), record.value()); }
@Override public void readMessage(ConsumerRecord<?, ?> record) { try { byte[] recordArray = (byte[]) record.value(); JsonObject jsonObj = (new JsonParser()).parse(new String(recordArray, Charsets.UTF_8)).getAsJsonObject(); jsonObj.addProperty(KAFKA_TOPIC.getFieldName(), record.topic()); jsonObj.addProperty(KAFKA_PARTITION_ID.getFieldName(), record.partition()); jsonObj.addProperty(KAFKA_OFFSET.getFieldName(), record.offset()); jsonObj.addProperty(KAFKA_TIMESTAMP.getFieldName(), record.timestamp()); jsonObj.addProperty(KAFKA_MSG_KEY.getFieldName(), record.key() != null ? record.key().toString() : null); jsonReader.setSource(jsonObj.toString().getBytes(Charsets.UTF_8)); jsonReader.write(writer); } catch (IOException e) { throw UserException.dataReadError(e).message(e.getMessage()) .addContext("MessageReader", JsonMessageReader.class.getName()).build(logger); } }
@Override public DeserializationHandlerResponse handle( final ProcessorContext context, final ConsumerRecord<byte[], byte[]> record, final Exception exception ) { log.warn( "Exception caught during Deserialization, " + "taskId: {}, topic: {}, partition: {}, offset: {}", context.taskId(), record.topic(), record.partition(), record.offset(), exception ); StreamsErrorCollector.recordError(context.applicationId(), record.topic()); return DeserializationHandlerResponse.CONTINUE; }
@Test public void iterator() throws Exception { Map<TopicPartition, List<ConsumerRecord<Integer, String>>> records = new LinkedHashMap<>(); String topic = "topic"; records.put(new TopicPartition(topic, 0), new ArrayList<ConsumerRecord<Integer, String>>()); ConsumerRecord<Integer, String> record1 = new ConsumerRecord<>(topic, 1, 0, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 1, "value1"); ConsumerRecord<Integer, String> record2 = new ConsumerRecord<>(topic, 1, 1, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, 2, "value2"); records.put(new TopicPartition(topic, 1), Arrays.asList(record1, record2)); records.put(new TopicPartition(topic, 2), new ArrayList<ConsumerRecord<Integer, String>>()); ConsumerRecords<Integer, String> consumerRecords = new ConsumerRecords<>(records); Iterator<ConsumerRecord<Integer, String>> iter = consumerRecords.iterator(); int c = 0; for (; iter.hasNext(); c++) { ConsumerRecord<Integer, String> record = iter.next(); assertEquals(1, record.partition()); assertEquals(topic, record.topic()); assertEquals(c, record.offset()); } assertEquals(2, c); } }
private void handleParseFailure(final ConsumerRecord<byte[], byte[]> consumerRecord, final ProcessSession session, final Exception cause, final String message) { // If we are unable to parse the data, we need to transfer it to 'parse failure' relationship final Map<String, String> attributes = getAttributes(consumerRecord); attributes.put(KafkaProcessorUtils.KAFKA_OFFSET, String.valueOf(consumerRecord.offset())); attributes.put(KafkaProcessorUtils.KAFKA_PARTITION, String.valueOf(consumerRecord.partition())); attributes.put(KafkaProcessorUtils.KAFKA_TOPIC, consumerRecord.topic()); FlowFile failureFlowFile = session.create(); final byte[] value = consumerRecord.value(); if (value != null) { failureFlowFile = session.write(failureFlowFile, out -> out.write(value)); } failureFlowFile = session.putAllAttributes(failureFlowFile, attributes); final String transitUri = KafkaProcessorUtils.buildTransitURI(securityProtocol, bootstrapServers, consumerRecord.topic()); session.getProvenanceReporter().receive(failureFlowFile, transitUri); session.transfer(failureFlowFile, REL_PARSE_FAILURE); if (cause == null) { logger.error(message); } else { logger.error(message, cause); } session.adjustCounter("Parse Failures", 1, false); }
private void handleParseFailure(final ConsumerRecord<byte[], byte[]> consumerRecord, final ProcessSession session, final Exception cause, final String message) { // If we are unable to parse the data, we need to transfer it to 'parse failure' relationship final Map<String, String> attributes = getAttributes(consumerRecord); attributes.put(KafkaProcessorUtils.KAFKA_OFFSET, String.valueOf(consumerRecord.offset())); attributes.put(KafkaProcessorUtils.KAFKA_PARTITION, String.valueOf(consumerRecord.partition())); attributes.put(KafkaProcessorUtils.KAFKA_TOPIC, consumerRecord.topic()); FlowFile failureFlowFile = session.create(); final byte[] value = consumerRecord.value(); if (value != null) { failureFlowFile = session.write(failureFlowFile, out -> out.write(value)); } failureFlowFile = session.putAllAttributes(failureFlowFile, attributes); final String transitUri = KafkaProcessorUtils.buildTransitURI(securityProtocol, bootstrapServers, consumerRecord.topic()); session.getProvenanceReporter().receive(failureFlowFile, transitUri); session.transfer(failureFlowFile, REL_PARSE_FAILURE); if (cause == null) { logger.error(message); } else { logger.error(message, cause); } session.adjustCounter("Parse Failures", 1, false); }
@Test public void testWriteReadFields2() throws IOException { ConsumerRecord<byte[], byte[]> record = new ConsumerRecord("topic", 0, 3L, "key".getBytes(), "value".getBytes()); KafkaWritable kafkaWritable = new KafkaWritable(record.partition(), record.offset(), record.timestamp(), record.value(), "thisKey".getBytes()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream w = new DataOutputStream(baos); kafkaWritable.write(w); w.flush(); ByteArrayInputStream input = new ByteArrayInputStream(baos.toByteArray()); DataInputStream inputStream = new DataInputStream(input); KafkaWritable actualKafkaWritable = new KafkaWritable(); actualKafkaWritable.readFields(inputStream); Assert.assertEquals(kafkaWritable, actualKafkaWritable); }
@Override @SuppressWarnings("deprecation") public ConsumerRecords<String, String> onConsume(ConsumerRecords<String, String> records) { // This will ensure that we get the cluster metadata when onConsume is called for the first time // as subsequent compareAndSet operations will fail. CLUSTER_ID_BEFORE_ON_CONSUME.compareAndSet(NO_CLUSTER_ID, CLUSTER_META.get()); Map<TopicPartition, List<ConsumerRecord<String, String>>> recordMap = new HashMap<>(); for (TopicPartition tp : records.partitions()) { List<ConsumerRecord<String, String>> lst = new ArrayList<>(); for (ConsumerRecord<String, String> record: records.records(tp)) { lst.add(new ConsumerRecord<>(record.topic(), record.partition(), record.offset(), record.timestamp(), record.timestampType(), record.checksum(), record.serializedKeySize(), record.serializedValueSize(), record.key(), record.value().toUpperCase(Locale.ROOT))); } recordMap.put(tp, lst); } return new ConsumerRecords<String, String>(recordMap); }
@Test public void testWriteReadFields() throws IOException { ConsumerRecord<byte[], byte[]> record = new ConsumerRecord("topic", 0, 3L, "key".getBytes(), "value".getBytes()); KafkaWritable kafkaWritable = new KafkaWritable(record.partition(), record.offset(), record.timestamp(), record.value(), null); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream w = new DataOutputStream(baos); kafkaWritable.write(w); w.flush(); ByteArrayInputStream input = new ByteArrayInputStream(baos.toByteArray()); DataInputStream inputStream = new DataInputStream(input); KafkaWritable actualKafkaWritable = new KafkaWritable(); actualKafkaWritable.readFields(inputStream); Assert.assertEquals(kafkaWritable, actualKafkaWritable); }
private void handleParseException(ParseException pe, ConsumerRecord<byte[], byte[]> record) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (tuningConfig.isLogParseExceptions()) { log.error( pe, "Encountered parse exception on row from partition[%d] offset[%d]", record.partition(), record.offset() ); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
final T value = deserializer.deserialize( record.key(), record.value(), record.topic(), record.partition(), record.offset());
final T value = deserializer.deserialize( record.key(), record.value(), record.topic(), record.partition(), record.offset());
private void compareIterator(List<ConsumerRecord<byte[], byte[]>> expected, Iterator<ConsumerRecord<byte[], byte[]>> kafkaRecordIterator) { expected.forEach((expectedRecord) -> { Assert.assertTrue("Record with offset is missing" + expectedRecord.offset(), kafkaRecordIterator.hasNext()); ConsumerRecord record = kafkaRecordIterator.next(); Assert.assertEquals(expectedRecord.topic(), record.topic()); Assert.assertEquals(expectedRecord.partition(), record.partition()); Assert.assertEquals("Offsets not matching", expectedRecord.offset(), record.offset()); byte[] binaryExceptedValue = expectedRecord.value(); byte[] binaryExceptedKey = expectedRecord.key(); byte[] binaryValue = (byte[]) record.value(); byte[] binaryKey = (byte[]) record.key(); Assert.assertArrayEquals("Values not matching", binaryExceptedValue, binaryValue); Assert.assertArrayEquals("Keys not matching", binaryExceptedKey, binaryKey); }); Assert.assertFalse(kafkaRecordIterator.hasNext()); }
@Test public void testRecordReader() { List<KafkaWritable> serRecords = expectedRecords.stream() .map((consumerRecord) -> new KafkaWritable(consumerRecord.partition(), consumerRecord.offset(), consumerRecord.timestamp(), consumerRecord.value(), consumerRecord.key())) .collect(Collectors.toList()); KafkaRecordReader recordReader = new KafkaRecordReader(); TaskAttemptContext context = new TaskAttemptContextImpl(this.conf, new TaskAttemptID()); recordReader.initialize(new KafkaInputSplit(currentTopic, 0, 50L, 100L, null), context); for (int i = 50; i < 100; ++i) { KafkaWritable record = new KafkaWritable(); Assert.assertTrue(recordReader.next(null, record)); Assert.assertEquals(serRecords.get(i), record); } recordReader.close(); }
@Test @SuppressWarnings("deprecation") public void testOldConstructor() { String topic = "topic"; int partition = 0; long offset = 23; String key = "key"; String value = "value"; ConsumerRecord<String, String> record = new ConsumerRecord<>(topic, partition, offset, key, value); assertEquals(topic, record.topic()); assertEquals(partition, record.partition()); assertEquals(offset, record.offset()); assertEquals(key, record.key()); assertEquals(value, record.value()); assertEquals(TimestampType.NO_TIMESTAMP_TYPE, record.timestampType()); assertEquals(ConsumerRecord.NO_TIMESTAMP, record.timestamp()); assertEquals(ConsumerRecord.NULL_CHECKSUM, record.checksum()); assertEquals(ConsumerRecord.NULL_SIZE, record.serializedKeySize()); assertEquals(ConsumerRecord.NULL_SIZE, record.serializedValueSize()); assertEquals(Optional.empty(), record.leaderEpoch()); assertEquals(new RecordHeaders(), record.headers()); }