private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
/** * Determines the starting offset for each store SSP (based on {@link #getStartingOffset(SystemStreamPartition, SystemAdmin)}) and * registers it with the respective SystemConsumer for starting consumption. */ private void registerStartingOffsets() { for (Map.Entry<String, SystemStream> changelogSystemStreamEntry : changelogSystemStreams.entrySet()) { SystemStreamPartition systemStreamPartition = new SystemStreamPartition(changelogSystemStreamEntry.getValue(), taskModel.getChangelogPartition()); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(changelogSystemStreamEntry.getValue().getSystem()); SystemConsumer systemConsumer = systemConsumers.get(changelogSystemStreamEntry.getKey()); String offset = getStartingOffset(systemStreamPartition, systemAdmin); if (offset != null) { LOG.info("Registering change log consumer with offset " + offset + " for %" + systemStreamPartition); systemConsumer.register(systemStreamPartition, offset); } else { LOG.info("Skipping change log restoration for {} because stream appears to be empty (offset was null).", systemStreamPartition); taskStoresToRestore.remove(changelogSystemStreamEntry.getKey()); } } }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
private <T> List<T> consumeMessages(Set<SystemStreamPartition> sspsToPoll) { SystemConsumer systemConsumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry); // register the consumer for ssps for (SystemStreamPartition ssp : sspsToPoll) { systemConsumer.register(ssp, "0"); } return consumeMessages(systemConsumer, sspsToPoll); }
private List<IncomingMessageEnvelope> consumeRawMessages(Set<SystemStreamPartition> sspsToPoll) { SystemConsumer systemConsumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry); // register the consumer for ssps for (SystemStreamPartition ssp : sspsToPoll) { systemConsumer.register(ssp, "0"); } return consumeRawMessages(systemConsumer, sspsToPoll); }
@Test public void testCheckpointsAreReadFromOldestOffset() throws Exception { KafkaStreamSpec checkpointSpec = new KafkaStreamSpec(CHECKPOINT_TOPIC, CHECKPOINT_TOPIC, CHECKPOINT_SYSTEM, 1); Config mockConfig = mock(Config.class); when(mockConfig.get(JobConfig.SSP_GROUPER_FACTORY())).thenReturn(GROUPER_FACTORY_CLASS); // mock out a consumer that returns a single checkpoint IME SystemStreamPartition ssp = new SystemStreamPartition("system-1", "input-topic", new Partition(0)); SystemConsumer mockConsumer = newConsumer(ImmutableList.of( ImmutableList.of(newCheckpointEnvelope(TASK1, ssp, "0")))); String oldestOffset = "0"; SystemAdmin mockAdmin = newAdmin(oldestOffset, "1"); SystemFactory factory = newFactory(mock(SystemProducer.class), mockConsumer, mockAdmin); KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, true, mockConfig, mock(MetricsRegistry.class), new CheckpointSerde(), new KafkaCheckpointLogKeySerde()); checkpointManager.register(TASK1); // 1. verify that consumer.register is called only during checkpointManager.start. // 2. verify that consumer.register is called with the oldest offset. // 3. verify that no other operation on the CheckpointManager re-invokes register since start offsets are set during // register verify(mockConsumer, times(0)).register(CHECKPOINT_SSP, oldestOffset); checkpointManager.start(); verify(mockConsumer, times(1)).register(CHECKPOINT_SSP, oldestOffset); checkpointManager.readLastCheckpoint(TASK1); verify(mockConsumer, times(1)).register(CHECKPOINT_SSP, oldestOffset); }
@Test public void testNullMessageWithValidMessageKey() { final String messageKey = "validKey"; SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry); systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, messageKey, null)); SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry); Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT) .mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))) .collect(Collectors.toSet()); // register the consumer for ssps for (SystemStreamPartition ssp : sspsToPoll) { consumer.register(ssp, "0"); } List<IncomingMessageEnvelope> results = consumeRawMessages(consumer, sspsToPoll); assertEquals(1, results.size()); assertEquals(results.get(0).getKey(), messageKey); assertNull(results.get(0).getMessage()); }
@Test public void testSinglePartitionConsumptionHappyPath() throws Exception { int partitionId = 0; TestMetricsRegistry testMetrics = new TestMetricsRegistry(); SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(partitionId)); Config eventHubConfig = createEventHubConfig(); EventHubSystemFactory factory = new EventHubSystemFactory(); SystemConsumer consumer = factory.getConsumer(SYSTEM_NAME, eventHubConfig, testMetrics); consumer.register(ssp, EventHubSystemConsumer.START_OF_STREAM); consumer.start(); int numEvents = 0; int numRetries = 20; while (numRetries-- > 0) { List<IncomingMessageEnvelope> result = consumer.poll(Collections.singleton(ssp), 2000).get(ssp); numEvents = result == null ? 0 : result.size(); if (numEvents > 0) { EventHubIncomingMessageEnvelope eventData = (EventHubIncomingMessageEnvelope) result.get(0); System.out.println("System properties: " + eventData.getEventData().getSystemProperties()); System.out.println("Key: " + new String((byte[]) eventData.getKey())); System.out.println("Message: " + new String((byte[]) eventData.getMessage())); } System.out.println("Retries left: " + numRetries); } Assert.assertTrue(numEvents > 0); }
@Test public void testConsumerRespectsOffset() { PageViewEvent event = new PageViewEvent(TEST_MEMBER_X, PAGE_ID_X, System.currentTimeMillis()); PageViewEvent event1 = new PageViewEvent(TEST_MEMBER_Y, PAGE_ID_Y, System.currentTimeMillis()); produceMessages(event); SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry); Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT) .mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))) .collect(Collectors.toSet()); // register the consumer for ssps for (SystemStreamPartition ssp : sspsToPoll) { consumer.register(ssp, "0"); } List<PageViewEvent> results = consumeMessages(consumer, sspsToPoll); assertEquals(1, results.size()); assertTrue(results.contains(event)); // nothing to poll results = consumeMessages(consumer, sspsToPoll); assertEquals(0, results.size()); produceMessages(event1); // got new message. check if the offset has progressed results = consumeMessages(consumer, sspsToPoll); assertEquals(1, results.size()); assertTrue(results.contains(event1)); }