/** * Get partitions counts only. Should be more efficient then getSystemStreamMetadata, but if not implemented * revert to getSystemStreamMetadata. * @param streamNames set of streams to query. * @param cacheTTL cacheTTL to use if caching the values. * @return A map from stream name to SystemStreamMetadata for each stream * requested in the parameter set. */ default Map<String, SystemStreamMetadata> getSystemStreamPartitionCounts(Set<String> streamNames, long cacheTTL) { return getSystemStreamMetadata(streamNames); }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void start() { for (SystemAdmin systemAdmin : systemAdminMap.values()) { systemAdmin.start(); } }
String resumeOffset = admin.getOffsetsAfter(ImmutableMap.of(ssp, fileOffset)).get(ssp); if (admin.offsetComparator(oldestOffset, resumeOffset) <= 0) { startingOffset = resumeOffset; } else {
maxChangeLogStreamPartitions); systemAdmin.start(); if (systemAdmin.createStream(changelogSpec)) { LOG.info(String.format("created changelog stream %s.", systemStream.getStream())); } else { LOG.info(String.format("changelog stream %s already exists.", systemStream.getStream())); systemAdmin.validateStream(changelogSpec); StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.createStream(accesslogSpec); systemAdmin.validateStream(accesslogSpec); systemAdmin.stop(); });
systemAdmin.start(); systemAdmin.createStream(streamSpec); systemAdmin.stop();
public void stop() { for (SystemAdmin systemAdmin : systemAdminMap.values()) { systemAdmin.stop(); } }
/** * Given some SSPs, but missing metadata for one of the streams, getSSPMetadata should delegate to * getSystemStreamMetadata and only fill in results for the SSPs corresponding to streams with metadata. */ @Test public void testGetSSPMetadataMissingStream() { SystemStreamPartition streamPartition0 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(0)); SystemStreamPartition otherStreamPartition0 = new SystemStreamPartition(SYSTEM, OTHER_STREAM, new Partition(0)); SystemAdmin systemAdmin = mock(MySystemAdmin.class); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("1", "2", "3"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of(new Partition(0), streamPartition0Metadata)))); Set<SystemStreamPartition> ssps = ImmutableSet.of(streamPartition0, otherStreamPartition0); when(systemAdmin.getSSPMetadata(ssps)).thenCallRealMethod(); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> expected = ImmutableMap.of(streamPartition0, streamPartition0Metadata); assertEquals(expected, systemAdmin.getSSPMetadata(ssps)); verify(systemAdmin).getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM)); }
private void maybeRefreshMetadata(SystemStreamPartition requestedSSP) { synchronized (this.metadataRefreshLock) { Instant refreshRequestedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); if (shouldRefresh(requestedSSP, refreshRequestedAt)) { String system = requestedSSP.getSystem(); Set<SystemStreamPartition> sspsToFetchFor = new HashSet<>(); sspsToFetchFor.add(requestedSSP); for (SystemStreamPartition sspToPrefetch : this.sspsToPrefetch) { if (system.equals(sspToPrefetch.getSystem()) && shouldRefresh(sspToPrefetch, refreshRequestedAt)) { sspsToFetchFor.add(sspToPrefetch); } } SystemAdmin systemAdmin = this.systemAdmins.getSystemAdmin(system); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> fetchedMetadata = systemAdmin.getSSPMetadata(sspsToFetchFor); Instant updatedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); // we want to add an entry even if there was no metadata, so iterate over sspsToFetchFor sspsToFetchFor.forEach(ssp -> this.cache.put(ssp, new CacheEntry(fetchedMetadata.get(ssp), updatedAt))); } } }
@Test public void testCreateCoordinatorStream() { SystemAdmin admin = Mockito.spy(systemAdmin()); StreamSpec spec = StreamSpec.createCoordinatorStreamSpec("testCoordinatorStream", "testSystem"); admin.createStream(spec); admin.validateStream(spec); Mockito.verify(admin).createStream(Mockito.any()); }
verify(admin1).clearStream(captor.capture()); assertEquals(captor.getValue().getPhysicalName(), STREAM1 + "-" + runId); verify(admin2).clearStream(captor.capture()); assertEquals(captor.getValue().getPhysicalName(), STREAM2 + "-" + runId);
/** * Validates each changelog system-stream with its respective SystemAdmin. */ private void validateChangelogStreams() { LOG.info("Validating change log streams: " + changelogSystemStreams); for (SystemStream changelogSystemStream : changelogSystemStreams.values()) { SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(changelogSystemStream.getSystem()); StreamSpec changelogSpec = StreamSpec.createChangeLogStreamSpec(changelogSystemStream.getStream(), changelogSystemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.validateStream(changelogSpec); } }
maxChangeLogStreamPartitions); systemAdmin.start(); if (systemAdmin.createStream(changelogSpec)) { LOG.info(String.format("created changelog stream %s.", systemStream.getStream())); } else { LOG.info(String.format("changelog stream %s already exists.", systemStream.getStream())); systemAdmin.validateStream(changelogSpec); StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.createStream(accesslogSpec); systemAdmin.validateStream(accesslogSpec); systemAdmin.stop(); });
systemAdmin.start(); systemAdmin.createStream(streamSpec); systemAdmin.stop();
public void stop() { for (SystemAdmin systemAdmin : systemAdminMap.values()) { systemAdmin.stop(); } }
/** * Given some SSPs, but missing metadata for one of the SSPs, getSSPMetadata should delegate to * getSystemStreamMetadata and only fill in results for the SSPs that have metadata. */ @Test public void testGetSSPMetadataMissingPartition() { SystemStreamPartition streamPartition0 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(0)); SystemStreamPartition streamPartition1 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(1)); SystemAdmin systemAdmin = mock(MySystemAdmin.class); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("1", "2", "3"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of(new Partition(0), streamPartition0Metadata)))); Set<SystemStreamPartition> ssps = ImmutableSet.of(streamPartition0, streamPartition1); when(systemAdmin.getSSPMetadata(ssps)).thenCallRealMethod(); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> expected = ImmutableMap.of(streamPartition0, streamPartition0Metadata); assertEquals(expected, systemAdmin.getSSPMetadata(ssps)); verify(systemAdmin).getSystemStreamMetadata(ImmutableSet.of(STREAM)); }
String resumeOffset = admin.getOffsetsAfter(ImmutableMap.of(ssp, fileOffset)).get(ssp); if (admin.offsetComparator(oldestOffset, resumeOffset) <= 0) { startingOffset = resumeOffset; } else {
private void maybeRefreshMetadata(SystemStreamPartition requestedSSP) { synchronized (this.metadataRefreshLock) { Instant refreshRequestedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); if (shouldRefresh(requestedSSP, refreshRequestedAt)) { String system = requestedSSP.getSystem(); Set<SystemStreamPartition> sspsToFetchFor = new HashSet<>(); sspsToFetchFor.add(requestedSSP); for (SystemStreamPartition sspToPrefetch : this.sspsToPrefetch) { if (system.equals(sspToPrefetch.getSystem()) && shouldRefresh(sspToPrefetch, refreshRequestedAt)) { sspsToFetchFor.add(sspToPrefetch); } } SystemAdmin systemAdmin = this.systemAdmins.getSystemAdmin(system); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> fetchedMetadata = systemAdmin.getSSPMetadata(sspsToFetchFor); Instant updatedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); // we want to add an entry even if there was no metadata, so iterate over sspsToFetchFor sspsToFetchFor.forEach(ssp -> this.cache.put(ssp, new CacheEntry(fetchedMetadata.get(ssp), updatedAt))); } } }
intStreams.forEach(stream -> { LOGGER.info("Clear intermediate stream {} in system {}", stream.getPhysicalName(), stream.getSystemName()); systemAdmins.getSystemAdmin(stream.getSystemName()).clearStream(stream); }); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(changelog); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), 1); systemAdmins.getSystemAdmin(spec.getSystemName()).clearStream(spec);
@Test(expected = StreamValidationException.class) public void testStartFailsOnTopicValidationErrors() { KafkaStreamSpec checkpointSpec = new KafkaStreamSpec(CHECKPOINT_TOPIC, CHECKPOINT_TOPIC, CHECKPOINT_SYSTEM, 1); // create an admin that throws an exception during validateStream SystemAdmin mockAdmin = newAdmin("0", "10"); doThrow(new StreamValidationException("invalid stream")).when(mockAdmin).validateStream(checkpointSpec); SystemFactory factory = newFactory(mock(SystemProducer.class), mock(SystemConsumer.class), mockAdmin); KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, true, mock(Config.class), mock(MetricsRegistry.class), null, new KafkaCheckpointLogKeySerde()); // expect an exception during startup checkpointManager.createResources(); checkpointManager.start(); }