@Override public Headers add(String key, byte[] value) throws IllegalStateException { return add(new RecordHeader(key, value)); }
private static Header[] readHeaders(ByteBuffer buffer, int numHeaders) { Header[] headers = new Header[numHeaders]; for (int i = 0; i < numHeaders; i++) { int headerKeySize = ByteUtils.readVarint(buffer); if (headerKeySize < 0) throw new InvalidRecordException("Invalid negative header key size " + headerKeySize); String headerKey = Utils.utf8(buffer, headerKeySize); buffer.position(buffer.position() + headerKeySize); ByteBuffer headerValue = null; int headerValueSize = ByteUtils.readVarint(buffer); if (headerValueSize >= 0) { headerValue = buffer.slice(); headerValue.limit(headerValueSize); buffer.position(buffer.position() + headerValueSize); } headers[i] = new RecordHeader(headerKey, headerValue); } return headers; }
@Override public EasyTransMsgPublishResult publish(String topic, String tag, String key, Map<String,Object> header, byte[] msgByte) { String kafkaTopic = QueueKafkaHelper.getKafkaTopic(topic, tag); //calculate partition TransactionId trxId = (TransactionId) header.get(EasytransConstant.CallHeadKeys.PARENT_TRX_ID_KEY); int partition = calcMessagePartition(kafkaTopic, trxId); List<Header> kafkaHeaderList = new ArrayList<>(header.size()); for(Entry<String, Object> entry:header.entrySet()){ kafkaHeaderList.add(new RecordHeader(entry.getKey(),serializer.serialization(entry.getValue()))); } ProducerRecord<String, byte[]> record = new ProducerRecord<>(kafkaTopic, partition, null, key, msgByte, kafkaHeaderList); Future<RecordMetadata> sendResultFuture = kafkaProducer.send(record); try { RecordMetadata recordMetadata = sendResultFuture.get(); log.info("message sent:" + recordMetadata); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException("message sent error",e); } EasyTransMsgPublishResult easyTransMsgPublishResult = new EasyTransMsgPublishResult(); easyTransMsgPublishResult.setTopic(topic); easyTransMsgPublishResult.setMessageId(key); return easyTransMsgPublishResult; }
@Test public void testLastHeader() { Headers headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); headers.add(new RecordHeader("key", "value2".getBytes())); headers.add(new RecordHeader("key", "value3".getBytes())); assertHeader("key", "value3", headers.lastHeader("key")); assertEquals(3, getCount(headers)); }
@Test public void testSizeInBytes() { Header[] headers = new Header[] { new RecordHeader("foo", "value".getBytes()), new RecordHeader("bar", (byte[]) null) }; long timestamp = System.currentTimeMillis(); SimpleRecord[] records = new SimpleRecord[] { new SimpleRecord(timestamp, "key".getBytes(), "value".getBytes()), new SimpleRecord(timestamp + 30000, null, "value".getBytes()), new SimpleRecord(timestamp + 60000, "key".getBytes(), null), new SimpleRecord(timestamp + 60000, "key".getBytes(), "value".getBytes(), headers) }; int actualSize = MemoryRecords.withRecords(CompressionType.NONE, records).sizeInBytes(); assertEquals(actualSize, DefaultRecordBatch.sizeInBytes(Arrays.asList(records))); }
@Test public void testHeaders() throws IOException { RecordHeaders headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); headers.add(new RecordHeader("key1", "key1value".getBytes())); headers.add(new RecordHeader("key", "value2".getBytes())); headers.add(new RecordHeader("key2", "key2value".getBytes())); Iterator<Header> keyHeaders = headers.headers("key").iterator(); assertHeader("key", "value", keyHeaders.next()); assertHeader("key", "value2", keyHeaders.next()); assertFalse(keyHeaders.hasNext()); keyHeaders = headers.headers("key1").iterator(); assertHeader("key1", "key1value", keyHeaders.next()); assertFalse(keyHeaders.hasNext()); keyHeaders = headers.headers("key2").iterator(); assertHeader("key2", "key2value", keyHeaders.next()); assertFalse(keyHeaders.hasNext()); }
headerList.add(new RecordHeader(RETRY_COUNT_KEY, serializer.serialization(retryCount))); headerList.add(new RecordHeader(ORGINAL_TOPIC, serializer.serialization(consumeRecord.topic())));
@Test public void testAdd() { Headers headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); Header header = headers.iterator().next(); assertHeader("key", "value", header); headers.add(new RecordHeader("key2", "value2".getBytes())); assertHeader("key2", "value2", headers.lastHeader("key2")); assertEquals(2, getCount(headers)); }
private void doTestConversion(boolean testConversionOverflow) throws IOException { List<Long> offsets = asList(0L, 2L, 3L, 9L, 11L, 15L, 16L, 17L, 22L, 24L); Header[] headers = {new RecordHeader("headerKey1", "headerValue1".getBytes()), new RecordHeader("headerKey2", "headerValue2".getBytes()), new RecordHeader("headerKey3", "headerValue3".getBytes())};
record.headers().add(new RecordHeader("test", "header2".getBytes())); producer.send(record, null); record.headers().add(new RecordHeader("test", "test".getBytes())); fail("Expected IllegalStateException to be raised"); } catch (IllegalStateException ise) {
@Test(expected = InvalidRecordException.class) public void testBasicSerdeInvalidHeaderCountTooLow() throws IOException { Header[] headers = new Header[] { new RecordHeader("foo", "value".getBytes()), new RecordHeader("bar", (byte[]) null), new RecordHeader("\"A\\u00ea\\u00f1\\u00fcC\"", "value".getBytes()) }; SimpleRecord record = new SimpleRecord(15L, "hi".getBytes(), "there".getBytes(), headers); int baseSequence = 723; long baseOffset = 37; int offsetDelta = 10; long baseTimestamp = System.currentTimeMillis(); long timestampDelta = 323; ByteBufferOutputStream out = new ByteBufferOutputStream(1024); DefaultRecord.writeTo(new DataOutputStream(out), offsetDelta, timestampDelta, record.key(), record.value(), record.headers()); ByteBuffer buffer = out.buffer(); buffer.flip(); buffer.put(14, (byte) 4); DefaultRecord logRecord = DefaultRecord.readFrom(buffer, baseOffset, baseTimestamp, baseSequence, null); // force iteration through the record to validate the number of headers assertEquals(DefaultRecord.sizeInBytes(offsetDelta, timestampDelta, record.key(), record.value(), record.headers()), logRecord.sizeInBytes()); }
@Test public void testRemove() { Headers headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); assertTrue(headers.iterator().hasNext()); headers.remove("key"); assertFalse(headers.iterator().hasNext()); }
@Test(expected = InvalidRecordException.class) public void testBasicSerdeInvalidHeaderCountTooHigh() throws IOException { Header[] headers = new Header[] { new RecordHeader("foo", "value".getBytes()), new RecordHeader("bar", (byte[]) null), new RecordHeader("\"A\\u00ea\\u00f1\\u00fcC\"", "value".getBytes()) }; SimpleRecord record = new SimpleRecord(15L, "hi".getBytes(), "there".getBytes(), headers); int baseSequence = 723; long baseOffset = 37; int offsetDelta = 10; long baseTimestamp = System.currentTimeMillis(); long timestampDelta = 323; ByteBufferOutputStream out = new ByteBufferOutputStream(1024); DefaultRecord.writeTo(new DataOutputStream(out), offsetDelta, timestampDelta, record.key(), record.value(), record.headers()); ByteBuffer buffer = out.buffer(); buffer.flip(); buffer.put(14, (byte) 8); DefaultRecord logRecord = DefaultRecord.readFrom(buffer, baseOffset, baseTimestamp, baseSequence, null); // force iteration through the record to validate the number of headers assertEquals(DefaultRecord.sizeInBytes(offsetDelta, timestampDelta, record.key(), record.value(), record.headers()), logRecord.sizeInBytes()); }
private void doTestConversion(CompressionType compressionType, byte toMagic) throws IOException { List<Long> offsets = asList(0L, 2L, 3L, 9L, 11L, 15L, 16L, 17L, 22L, 24L); Header[] headers = {new RecordHeader("headerKey1", "headerValue1".getBytes()), new RecordHeader("headerKey2", "headerValue2".getBytes()), new RecordHeader("headerKey3", "headerValue3".getBytes())};
@Test public void testReadOnly() throws IOException { RecordHeaders headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); Iterator<Header> headerIteratorBeforeClose = headers.iterator(); headers.setReadOnly(); try { headers.add(new RecordHeader("key", "value".getBytes())); fail("IllegalStateException expected as headers are closed"); } catch (IllegalStateException ise) {
@Test public void testNew() throws IOException { RecordHeaders headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); headers.setReadOnly(); RecordHeaders newHeaders = new RecordHeaders(headers); newHeaders.add(new RecordHeader("key", "value2".getBytes())); //Ensure existing headers are not modified assertHeader("key", "value", headers.lastHeader("key")); assertEquals(1, getCount(headers)); //Ensure new headers are modified assertHeader("key", "value2", newHeaders.lastHeader("key")); assertEquals(2, getCount(newHeaders)); }
headersArray[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8)); builder.append(0L, "key".getBytes(), "value-2".getBytes(), headersArray); headersArray2[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8)); headersArray2[1] = new RecordHeader("headerKey", "headerValue2".getBytes(StandardCharsets.UTF_8)); builder.append(0L, "key".getBytes(), "value-3".getBytes(), headersArray2);
0L); ProducerBatch batch = new ProducerBatch(new TopicPartition("topic", 1), builder, now); Header header = new RecordHeader("header-key", "header-value".getBytes());
@Test public void testBasicSerde() throws IOException { Header[] headers = new Header[] { new RecordHeader("foo", "value".getBytes()), new RecordHeader("bar", (byte[]) null), new RecordHeader("\"A\\u00ea\\u00f1\\u00fcC\"", "value".getBytes()) };
@Test public void testAddRemoveInterleaved() { Headers headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); headers.add(new RecordHeader("key2", "value2".getBytes())); headers.add(new RecordHeader("key3", "value3".getBytes())); headers.add(new RecordHeader("key3", "value4".getBytes())); headers.add(new RecordHeader("key", "valueNew".getBytes()));