/** * Shutdown this ShardConsumer (including invoking the RecordProcessor shutdown API). * This is called by Worker when it loses responsibility for a shard. * * @return true if shutdown is complete (false if shutdown is still in progress) */ synchronized boolean beginShutdown() { markForShutdown(ShutdownReason.ZOMBIE); checkAndSubmitNextTask(); return isShutdown(); }
@Override public ITask createTask(ShardConsumer consumer) { return new InitializeTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getCheckpoint(), consumer.getRecordProcessorCheckpointer(), consumer.getDataFetcher(), consumer.getTaskBackoffTimeMillis(), consumer.getStreamConfig(), consumer.getGetRecordsCache()); }
@Override public ITask createTask(ShardConsumer consumer) { return new ShutdownTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getShutdownReason(), consumer.getStreamConfig().getStreamProxy(), consumer.getStreamConfig().getInitialPositionInStream(), consumer.isCleanupLeasesOfCompletedShards(), consumer.isIgnoreUnexpectedChildShards(), consumer.getLeaseManager(), consumer.getTaskBackoffTimeMillis(), consumer.getGetRecordsCache()); }
@Override public ITask createTask(ShardConsumer consumer) { return new ProcessTask(consumer.getShardInfo(), consumer.getStreamConfig(), consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getDataFetcher(), consumer.getTaskBackoffTimeMillis(), consumer.isSkipShardSyncAtWorkerInitializationIfLeasesExist(), consumer.getGetRecordsCache()); }
private synchronized boolean checkAndSubmitNextTask() { boolean submittedNewTask = false; if (readyForNextTask()) { TaskOutcome taskOutcome = TaskOutcome.NOT_COMPLETE; if (future != null && future.isDone()) { taskOutcome = determineTaskOutcome(); updateState(taskOutcome); ITask nextTask = getNextTask(); if (nextTask != null) { currentTask = nextTask;
@Override public ITask createTask(ShardConsumer consumer) { return new BlockOnParentShardTask(consumer.getShardInfo(), consumer.getLeaseManager(), consumer.getParentShardPollIntervalMillis()); }
for (ShardInfo shardInfo : getShardInfoForAssignments()) { ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); if (shardConsumer.isShutdown() && shardConsumer.getShutdownReason().equals(ShutdownReason.TERMINATE)) { foundCompletedShard = true; } else { shardConsumer.consumeShard();
/** * No-op if current task is pending, otherwise submits next task for this shard. * This method should NOT be called if the ShardConsumer is already in SHUTDOWN_COMPLETED state. * * @return true if a new process task was submitted, false otherwise */ synchronized boolean consumeShard() { return checkAndSubmitNextTask(); }
/** * NOTE: This method is internal/private to the Worker class. It has package access solely for testing. * * This method relies on ShardInfo.equals() method returning true for ShardInfo objects which may have been * instantiated with parentShardIds in a different order (and rest of the fields being the equal). For example * shardInfo1.equals(shardInfo2) should return true with shardInfo1 and shardInfo2 defined as follows. ShardInfo * shardInfo1 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent1", "parent2")); ShardInfo * shardInfo2 = new ShardInfo(shardId1, concurrencyToken1, Arrays.asList("parent2", "parent1")); */ void cleanupShardConsumers(Set<ShardInfo> assignedShards) { for (ShardInfo shard : shardInfoShardConsumerMap.keySet()) { if (!assignedShards.contains(shard)) { // Shutdown the consumer since we are no longer responsible for // the shard. boolean isShutdown = shardInfoShardConsumerMap.get(shard).beginShutdown(); if (isShutdown) { shardInfoShardConsumerMap.remove(shard); } } } }
protected ShardConsumer buildConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { IRecordProcessor recordProcessor = processorFactory.createProcessor(); return new ShardConsumer(shardInfo, streamConfig, checkpointTracker, recordProcessor, leaseCoordinator.getLeaseManager(), parentShardPollIntervalMillis, cleanupLeasesUponShardCompletion, executorService, metricsFactory, taskBackoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, retryGetRecordsInSeconds, maxGetRecordsThreadPool, config); }