/** * Emits metrics, and sleeps if there are no records available * * @param startTimeMillis * the time when the task started */ private void handleNoRecords(long startTimeMillis) { LOG.debug("Kinesis didn't return any records for shard " + shardInfo.getShardId()); long sleepTimeMillis = streamConfig.getIdleTimeInMilliseconds() - (System.currentTimeMillis() - startTimeMillis); if (sleepTimeMillis > 0) { sleepTimeMillis = Math.max(sleepTimeMillis, streamConfig.getIdleTimeInMilliseconds()); try { LOG.debug("Sleeping for " + sleepTimeMillis + " ms since there were no new records in shard " + shardInfo.getShardId()); Thread.sleep(sleepTimeMillis); } catch (InterruptedException e) { LOG.debug("ShardId " + shardInfo.getShardId() + ": Sleep was interrupted"); } } }
this.backoffTimeMillis = backoffTimeMillis; this.throttlingReporter = throttlingReporter; IKinesisProxy kinesisProxy = this.streamConfig.getStreamProxy(); this.getRecordsCache = getRecordsCache;
/** * Whether we should call process records or not * * @param records * the records returned from the call to Kinesis, and/or deaggregation * @return true if the set of records should be dispatched to the record process, false if they should not. */ private boolean shouldCallProcessRecords(List<Record> records) { return (!records.isEmpty()) || streamConfig.shouldCallProcessRecordsEvenForEmptyRecordList(); }
this.cleanupLeasesUponShardCompletion = cleanupLeasesUponShardCompletion; this.checkpointTracker = checkpoint != null ? checkpoint : leaseCoordinator; this.idleTimeInMilliseconds = streamConfig.getIdleTimeInMilliseconds(); this.executorService = execService; this.leaseCoordinator = leaseCoordinator; this.metricsFactory = metricsFactory; this.controlServer = new ShardSyncTaskManager(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), initialPositionInStream, cleanupLeasesUponShardCompletion, config.shouldIgnoreUnexpectedChildShards(), shardSyncIdleTimeMillis, metricsFactory, executorService);
checkpoint, new SequenceNumberValidator( streamConfig.getStreamProxy(), shardInfo.getShardId(), streamConfig.shouldValidateSequenceNumberBeforeCheckpointing()), metricsFactory), leaseManager, backoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, new KinesisDataFetcher(streamConfig.getStreamProxy(), shardInfo), retryGetRecordsInSeconds, maxGetRecordsThreadPool,
@Override public ITask createTask(ShardConsumer consumer) { return new ShutdownTask(consumer.getShardInfo(), consumer.getRecordProcessor(), consumer.getRecordProcessorCheckpointer(), consumer.getShutdownReason(), consumer.getStreamConfig().getStreamProxy(), consumer.getStreamConfig().getInitialPositionInStream(), consumer.isCleanupLeasesOfCompletedShards(), consumer.isIgnoreUnexpectedChildShards(), consumer.getLeaseManager(), consumer.getTaskBackoffTimeMillis(), consumer.getGetRecordsCache()); }
.getSequenceNumber(), streamConfig.getInitialPositionInStream());
recordProcessorFactory, config, new StreamConfig(kinesisProxy, config.getMaxRecords(), config.getIdleTimeBetweenReadsInMillis(),
ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.getCheckpoint(); dataFetcher.initialize(initialCheckpoint.getSequenceNumber(), streamConfig.getInitialPositionInStream()); getRecordsCache.start(); recordProcessorCheckpointer.setLargestPermittedCheckpointValue(initialCheckpoint);
this(config.getApplicationName(), new V1ToV2RecordProcessorFactoryAdapter(recordProcessorFactory), config, new StreamConfig( new KinesisProxy(config, kinesisClient), config.getMaxRecords(), config.getIdleTimeBetweenReadsInMillis(),
|| leaseCoordinator.getLeaseManager().isLeaseTableEmpty()) { LOG.info("Syncing Kinesis shard info"); ShardSyncTask shardSyncTask = new ShardSyncTask(streamConfig.getStreamProxy(), leaseCoordinator.getLeaseManager(), initialPosition, cleanupLeasesUponShardCompletion, config.shouldIgnoreUnexpectedChildShards(), 0L);