/** * Get partitions counts only. Should be more efficient then getSystemStreamMetadata, but if not implemented * revert to getSystemStreamMetadata. * @param streamNames set of streams to query. * @param cacheTTL cacheTTL to use if caching the values. * @return A map from stream name to SystemStreamMetadata for each stream * requested in the parameter set. */ default Map<String, SystemStreamMetadata> getSystemStreamPartitionCounts(Set<String> streamNames, long cacheTTL) { return getSystemStreamMetadata(streamNames); }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
/** * Fetch metadata from a system for a set of SSPs. * Implementors should override this if there is a more efficient implementation than delegating to * {@link #getSystemStreamMetadata}. * * @param ssps SSPs for which to get metadata * @return A map from SystemStreamPartition to the SystemStreamPartitionMetadata, with an entry for each SSP in * {@code ssps} for which metadata could be found * @throws RuntimeException if there was an error fetching metadata */ default Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> getSSPMetadata( Set<SystemStreamPartition> ssps) { Set<String> streams = ssps.stream().map(SystemStream::getStream).collect(Collectors.toSet()); Map<String, SystemStreamMetadata> streamToSystemStreamMetadata = getSystemStreamMetadata(streams); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> sspToSSPMetadata = new HashMap<>(); for (SystemStreamPartition ssp : ssps) { SystemStreamMetadata systemStreamMetadata = streamToSystemStreamMetadata.get(ssp.getStream()); if (systemStreamMetadata != null) { SystemStreamMetadata.SystemStreamPartitionMetadata sspMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(ssp.getPartition()); if (sspMetadata != null) { sspToSSPMetadata.put(ssp, sspMetadata); } } } return sspToSSPMetadata; }
/** * Fetch metadata from a system for a set of SSPs. * Implementors should override this if there is a more efficient implementation than delegating to * {@link #getSystemStreamMetadata}. * * @param ssps SSPs for which to get metadata * @return A map from SystemStreamPartition to the SystemStreamPartitionMetadata, with an entry for each SSP in * {@code ssps} for which metadata could be found * @throws RuntimeException if there was an error fetching metadata */ default Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> getSSPMetadata( Set<SystemStreamPartition> ssps) { Set<String> streams = ssps.stream().map(SystemStream::getStream).collect(Collectors.toSet()); Map<String, SystemStreamMetadata> streamToSystemStreamMetadata = getSystemStreamMetadata(streams); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> sspToSSPMetadata = new HashMap<>(); for (SystemStreamPartition ssp : ssps) { SystemStreamMetadata systemStreamMetadata = streamToSystemStreamMetadata.get(ssp.getStream()); if (systemStreamMetadata != null) { SystemStreamMetadata.SystemStreamPartitionMetadata sspMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(ssp.getPartition()); if (sspMetadata != null) { sspToSSPMetadata.put(ssp, sspMetadata); } } } return sspToSSPMetadata; }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Create a new {@link SystemAdmin} that returns the provided oldest and newest offsets for its topics */ private SystemAdmin newAdmin(String oldestOffset, String newestOffset) { SystemStreamMetadata checkpointTopicMetadata = new SystemStreamMetadata(CHECKPOINT_TOPIC, ImmutableMap.of(new Partition(0), new SystemStreamPartitionMetadata(oldestOffset, newestOffset, Integer.toString(Integer.parseInt(newestOffset) + 1)))); SystemAdmin mockAdmin = mock(SystemAdmin.class); when(mockAdmin.getSystemStreamMetadata(Collections.singleton(CHECKPOINT_TOPIC))).thenReturn( ImmutableMap.of(CHECKPOINT_TOPIC, checkpointTopicMetadata)); return mockAdmin; }
@Test public void testGetStreamPartitionCounts() { SystemAdmin admin1 = mock(SystemAdmin.class); SystemAdmins systemAdmins = mock(SystemAdmins.class); when(systemAdmins.getSystemAdmin(SYSTEM1)).thenReturn(admin1); Map<String, SystemStreamMetadata> map = new HashMap<>(); SystemStreamMetadata meta1 = mock(SystemStreamMetadata.class); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitions = new HashMap<>(); partitions.put(new Partition(0), null); when(meta1.getSystemStreamPartitionMetadata()).thenReturn(partitions); map.put(STREAM1, meta1); SystemStreamMetadata meta2 = mock(SystemStreamMetadata.class); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitions2 = new HashMap<>(); partitions2.put(new Partition(0), null); partitions2.put(new Partition(1), null); when(meta2.getSystemStreamPartitionMetadata()).thenReturn(partitions2); map.put(STREAM2, meta2); when(admin1.getSystemStreamMetadata(anyObject())).thenReturn(map); Set<String> streams = new HashSet<>(); streams.add(STREAM1); streams.add(STREAM2); StreamManager manager = new StreamManager(systemAdmins); Map<String, Integer> counts = manager.getStreamPartitionCounts(SYSTEM1, streams); assertTrue(counts.get(STREAM1).equals(1)); assertTrue(counts.get(STREAM2).equals(2)); }
/** * Given some SSPs, but missing metadata for one of the streams, getSSPMetadata should delegate to * getSystemStreamMetadata and only fill in results for the SSPs corresponding to streams with metadata. */ @Test public void testGetSSPMetadataMissingStream() { SystemStreamPartition streamPartition0 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(0)); SystemStreamPartition otherStreamPartition0 = new SystemStreamPartition(SYSTEM, OTHER_STREAM, new Partition(0)); SystemAdmin systemAdmin = mock(MySystemAdmin.class); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("1", "2", "3"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of(new Partition(0), streamPartition0Metadata)))); Set<SystemStreamPartition> ssps = ImmutableSet.of(streamPartition0, otherStreamPartition0); when(systemAdmin.getSSPMetadata(ssps)).thenCallRealMethod(); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> expected = ImmutableMap.of(streamPartition0, streamPartition0Metadata); assertEquals(expected, systemAdmin.getSSPMetadata(ssps)); verify(systemAdmin).getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM)); }
/** * Given some SSPs, but missing metadata for one of the SSPs, getSSPMetadata should delegate to * getSystemStreamMetadata and only fill in results for the SSPs that have metadata. */ @Test public void testGetSSPMetadataMissingPartition() { SystemStreamPartition streamPartition0 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(0)); SystemStreamPartition streamPartition1 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(1)); SystemAdmin systemAdmin = mock(MySystemAdmin.class); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("1", "2", "3"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of(new Partition(0), streamPartition0Metadata)))); Set<SystemStreamPartition> ssps = ImmutableSet.of(streamPartition0, streamPartition1); when(systemAdmin.getSSPMetadata(ssps)).thenCallRealMethod(); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> expected = ImmutableMap.of(streamPartition0, streamPartition0Metadata); assertEquals(expected, systemAdmin.getSSPMetadata(ssps)); verify(systemAdmin).getSystemStreamMetadata(ImmutableSet.of(STREAM)); }
@Ignore("Integration Test") @Test public void testGetStreamMetadata() { EventHubSystemFactory eventHubSystemFactory = new EventHubSystemFactory(); SystemAdmin eventHubSystemAdmin = eventHubSystemFactory.getAdmin(SYSTEM_NAME, MockEventHubConfigFactory.getEventHubConfig(EventHubSystemProducer.PartitioningMethod.EVENT_HUB_HASHING)); Set<String> streams = new HashSet<>(); streams.add(STREAM_NAME1); streams.add(STREAM_NAME2); Map<String, SystemStreamMetadata> metadataMap = eventHubSystemAdmin.getSystemStreamMetadata(streams); for (String stream : streams) { Assert.assertTrue(metadataMap.containsKey(stream)); Assert.assertEquals(stream, metadataMap.get(stream).getStreamName()); Assert.assertNotNull(metadataMap.get(stream).getSystemStreamPartitionMetadata()); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadataMap = metadataMap.get(stream).getSystemStreamPartitionMetadata(); Assert.assertTrue(partitionMetadataMap.size() >= MIN_EVENTHUB_ENTITY_PARTITION); Assert.assertTrue(partitionMetadataMap.size() <= MAX_EVENTHUB_ENTITY_PARTITION); partitionMetadataMap.forEach((partition, metadata) -> { Assert.assertEquals(EventHubSystemConsumer.START_OF_STREAM, metadata.getOldestOffset()); Assert.assertNotSame(EventHubSystemConsumer.END_OF_STREAM, metadata.getNewestOffset()); String expectedUpcomingOffset = String.valueOf(Long.parseLong(metadata.getNewestOffset()) + 1); Assert.assertEquals(expectedUpcomingOffset, metadata.getUpcomingOffset()); }); } }
SystemStreamMetadata.SystemStreamPartitionMetadata otherStreamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("21", "22", "23"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of( new Partition(0), streamPartition0Metadata, otherStreamPartition0, otherStreamPartition0Metadata); assertEquals(expected, systemAdmin.getSSPMetadata(ssps)); verify(systemAdmin).getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM));