@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return getSystemStreamMetadata(streamNames, new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS)); }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return getSystemStreamMetadata(streamNames, new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS)); }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return getSystemStreamMetadata(streamNames, new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS)); }
@Override public void validateStream(StreamSpec streamSpec) throws StreamValidationException { LOG.info("About to validate stream = " + streamSpec); String streamName = streamSpec.getPhysicalName(); SystemStreamMetadata systemStreamMetadata = getSystemStreamMetadata(Collections.singleton(streamName)).get(streamName); if (systemStreamMetadata == null) { throw new StreamValidationException( "Failed to obtain metadata for stream " + streamName + ". Validation failed."); } int actualPartitionCounter = systemStreamMetadata.getSystemStreamPartitionMetadata().size(); int expectedPartitionCounter = streamSpec.getPartitionCount(); LOG.info("actualCount=" + actualPartitionCounter + "; expectedCount=" + expectedPartitionCounter); if (actualPartitionCounter != expectedPartitionCounter) { throw new StreamValidationException( String.format("Mismatch of partitions for stream %s. Expected %d, got %d. Validation failed.", streamName, expectedPartitionCounter, actualPartitionCounter)); } }
@Override public void validateStream(StreamSpec streamSpec) throws StreamValidationException { LOG.info("About to validate stream = " + streamSpec); String streamName = streamSpec.getPhysicalName(); SystemStreamMetadata systemStreamMetadata = getSystemStreamMetadata(Collections.singleton(streamName)).get(streamName); if (systemStreamMetadata == null) { throw new StreamValidationException( "Failed to obtain metadata for stream " + streamName + ". Validation failed."); } int actualPartitionCounter = systemStreamMetadata.getSystemStreamPartitionMetadata().size(); int expectedPartitionCounter = streamSpec.getPartitionCount(); LOG.info("actualCount=" + actualPartitionCounter + "; expectedCount=" + expectedPartitionCounter); if (actualPartitionCounter != expectedPartitionCounter) { throw new StreamValidationException( String.format("Mismatch of partitions for stream %s. Expected %d, got %d. Validation failed.", streamName, expectedPartitionCounter, actualPartitionCounter)); } }
@Override public void validateStream(StreamSpec streamSpec) throws StreamValidationException { LOG.info("About to validate stream = " + streamSpec); String streamName = streamSpec.getPhysicalName(); SystemStreamMetadata systemStreamMetadata = getSystemStreamMetadata(Collections.singleton(streamName)).get(streamName); if (systemStreamMetadata == null) { throw new StreamValidationException( "Failed to obtain metadata for stream " + streamName + ". Validation failed."); } int actualPartitionCounter = systemStreamMetadata.getSystemStreamPartitionMetadata().size(); int expectedPartitionCounter = streamSpec.getPartitionCount(); LOG.info("actualCount=" + actualPartitionCounter + "; expectedCount=" + expectedPartitionCounter); if (actualPartitionCounter != expectedPartitionCounter) { throw new StreamValidationException( String.format("Mismatch of partitions for stream %s. Expected %d, got %d. Validation failed.", streamName, expectedPartitionCounter, actualPartitionCounter)); } }
@Test(expected = SamzaException.class) public void testGetSystemStreamMetadataShouldTerminateAfterFiniteRetriesOnException() { when(mockKafkaConsumer.partitionsFor(VALID_TOPIC)).thenThrow(new RuntimeException()) .thenThrow(new RuntimeException()) .thenThrow(new RuntimeException()) .thenThrow(new RuntimeException()) .thenThrow(new RuntimeException()); kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC)); }
@Test public void testGetSystemStreamMetaDataWithInvalidTopic() { Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(INVALID_TOPIC)); assertEquals("empty metadata for invalid topic", metadataMap.size(), 0); }
@Test public void testGetSystemStreamMetaDataWithNoTopic() { Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(Collections.emptySet()); assertEquals("empty metadata for no topic", metadataMap.size(), 0); }
@Test public void testGetSystemStreamMetaDataWithRetry() { final List<PartitionInfo> partitionInfosForTopic = ImmutableList.of(mockPartitionInfo0, mockPartitionInfo1); when(mockKafkaConsumer.partitionsFor(VALID_TOPIC)).thenThrow(new RuntimeException()) .thenReturn(partitionInfosForTopic); Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC)); assertEquals("metadata should return for 1 topic", metadataMap.size(), 1); // retried twice because the first fails and the second succeeds Mockito.verify(mockKafkaConsumer, Mockito.times(2)).partitionsFor(VALID_TOPIC); final List<TopicPartition> topicPartitions = Arrays.asList(new TopicPartition(mockPartitionInfo0.topic(), mockPartitionInfo0.partition()), new TopicPartition(mockPartitionInfo1.topic(), mockPartitionInfo1.partition())); // the following methods thereafter are only called once Mockito.verify(mockKafkaConsumer, Mockito.times(1)).beginningOffsets(topicPartitions); Mockito.verify(mockKafkaConsumer, Mockito.times(1)).endOffsets(topicPartitions); }
System.out.println("STARTING"); Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC));
@Test public void testGetSystemStreamMetaDataForTopicWithNoMessage() { // The topic with no messages will have beginningOffset = 0 and endOffset = 0 when(mockKafkaConsumer.beginningOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); when(mockKafkaConsumer.endOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC)); assertEquals("metadata should return for 1 topic", metadataMap.size(), 1); // verify the metadata streamName assertEquals("the stream name should be " + VALID_TOPIC, metadataMap.get(VALID_TOPIC).getStreamName(), VALID_TOPIC); // verify the offset for each partition Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> systemStreamPartitionMetadata = metadataMap.get(VALID_TOPIC).getSystemStreamPartitionMetadata(); assertEquals("there are 2 partitions", systemStreamPartitionMetadata.size(), 2); SystemStreamMetadata.SystemStreamPartitionMetadata partition0Metadata = systemStreamPartitionMetadata.get(new Partition(0)); assertEquals("oldest offset for partition 0", partition0Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 0", partition0Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition0Metadata.getNewestOffset(), null); SystemStreamMetadata.SystemStreamPartitionMetadata partition1Metadata = systemStreamPartitionMetadata.get(new Partition(1)); assertEquals("oldest offset for partition 1", partition1Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 1", partition1Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition1Metadata.getNewestOffset(), null); }