/** * Sleeps for the configured backoff period. This is usually only called when an exception occurs. */ private void backoff() { // backoff if we encounter an exception. try { Thread.sleep(this.backoffTimeMillis); } catch (InterruptedException ie) { LOG.debug(shardInfo.getShardId() + ": Sleep was interrupted", ie); } }
/** * * @param kinesisProxy Kinesis proxy * @param shardInfo The shardInfo object. */ public KinesisDataFetcher(IKinesisProxy kinesisProxy, ShardInfo shardInfo) { this.shardId = shardInfo.getShardId(); this.kinesisProxy = new MetricsCollectingKinesisProxyDecorator("KinesisDataFetcher", kinesisProxy, this.shardId); }
/** * Emits metrics, and sleeps if there are no records available * * @param startTimeMillis * the time when the task started */ private void handleNoRecords(long startTimeMillis) { LOG.debug("Kinesis didn't return any records for shard " + shardInfo.getShardId()); long sleepTimeMillis = streamConfig.getIdleTimeInMilliseconds() - (System.currentTimeMillis() - startTimeMillis); if (sleepTimeMillis > 0) { sleepTimeMillis = Math.max(sleepTimeMillis, streamConfig.getIdleTimeInMilliseconds()); try { LOG.debug("Sleeping for " + sleepTimeMillis + " ms since there were no new records in shard " + shardInfo.getShardId()); Thread.sleep(sleepTimeMillis); } catch (InterruptedException e) { LOG.debug("ShardId " + shardInfo.getShardId() + ": Sleep was interrupted"); } } }
private static final GetRecordsRetrievalStrategy makeStrategy(KinesisDataFetcher dataFetcher, Optional<Integer> retryGetRecordsInSeconds, Optional<Integer> maxGetRecordsThreadPool, ShardInfo shardInfo) { Optional<GetRecordsRetrievalStrategy> getRecordsRetrievalStrategy = retryGetRecordsInSeconds.flatMap(retry -> maxGetRecordsThreadPool.map(max -> new AsynchronousGetRecordsRetrievalStrategy(dataFetcher, retry, max, shardInfo.getShardId()))); return getRecordsRetrievalStrategy.orElse(new SynchronousGetRecordsRetrievalStrategy(dataFetcher)); }
/** * {@inheritDoc} */ @Override public synchronized void checkpoint() throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException { if (LOG.isDebugEnabled()) { LOG.debug("Checkpointing " + shardInfo.getShardId() + ", " + " token " + shardInfo.getConcurrencyToken() + " at largest permitted value " + this.largestPermittedCheckpointValue); } advancePosition(this.largestPermittedCheckpointValue); }
@Override public List<ShardInfo> prioritize(List<ShardInfo> original) { Map<String, ShardInfo> shards = new HashMap<>(); for (ShardInfo shardInfo : original) { shards.put(shardInfo.getShardId(), shardInfo); } Map<String, SortingNode> processedNodes = new HashMap<>(); for (ShardInfo shardInfo : original) { populateDepth(shardInfo.getShardId(), shards, processedNodes); } List<ShardInfo> orderedInfos = new ArrayList<>(original.size()); List<SortingNode> orderedNodes = new ArrayList<>(processedNodes.values()); Collections.sort(orderedNodes); for (SortingNode sortingTreeNode : orderedNodes) { // don't process shards with depth > maxDepth if (sortingTreeNode.getDepth() <= maxDepth) { orderedInfos.add(sortingTreeNode.shardInfo); } } return orderedInfos; }
/** * @param shardInfo * contains information about the shard * @param streamConfig * Stream configuration * @param recordProcessor * Record processor used to process the data records for the shard * @param recordProcessorCheckpointer * Passed to the RecordProcessor so it can checkpoint progress * @param dataFetcher * Kinesis data fetcher (used to fetch records from Kinesis) * @param backoffTimeMillis * backoff time when catching exceptions * @param getRecordsCache * The retrieval strategy for fetching records from kinesis */ public ProcessTask(ShardInfo shardInfo, StreamConfig streamConfig, IRecordProcessor recordProcessor, RecordProcessorCheckpointer recordProcessorCheckpointer, KinesisDataFetcher dataFetcher, long backoffTimeMillis, boolean skipShardSyncAtWorkerInitializationIfLeasesExist, GetRecordsCache getRecordsCache) { this(shardInfo, streamConfig, recordProcessor, recordProcessorCheckpointer, dataFetcher, backoffTimeMillis, skipShardSyncAtWorkerInitializationIfLeasesExist, new ThrottlingReporter(MAX_CONSECUTIVE_THROTTLES, shardInfo.getShardId()), getRecordsCache); }
private void logTaskException(TaskResult taskResult) { if (LOG.isDebugEnabled()) { Exception taskException = taskResult.getException(); if (taskException instanceof BlockedOnParentShardException) { // No need to log the stack trace for this exception (it is very specific). LOG.debug("Shard " + shardInfo.getShardId() + " is blocked on completion of parent shard."); } else { LOG.debug("Caught exception running " + currentTask.getTaskType() + " task: ", taskResult.getException()); } } }
/** * Dispatches a batch of records to the record processor, and handles any fallout from that. * * @param input * the result of the last call to Kinesis * @param records * the records to be dispatched. It's possible the records have been adjusted by KPL deaggregation. */ private void callProcessRecords(ProcessRecordsInput input, List<Record> records) { LOG.debug("Calling application processRecords() with " + records.size() + " records from " + shardInfo.getShardId()); final ProcessRecordsInput processRecordsInput = new ProcessRecordsInput().withRecords(records) .withCheckpointer(recordProcessorCheckpointer) .withMillisBehindLatest(input.getMillisBehindLatest()); final long recordProcessorStartTimeMillis = System.currentTimeMillis(); try { recordProcessor.processRecords(processRecordsInput); } catch (Exception e) { LOG.error("ShardId " + shardInfo.getShardId() + ": Application processRecords() threw an exception when processing shard ", e); LOG.error("ShardId " + shardInfo.getShardId() + ": Skipping over the following data records: " + records); } finally { MetricsHelper.addLatencyPerShard(shardInfo.getShardId(), RECORD_PROCESSOR_PROCESS_RECORDS_METRIC, recordProcessorStartTimeMillis, MetricsLevel.SUMMARY); } }
private List<ShardInfo> getShardInfoForAssignments() { List<ShardInfo> assignedStreamShards = leaseCoordinator.getCurrentAssignments(); List<ShardInfo> prioritizedShards = shardPrioritization.prioritize(assignedStreamShards); if ((prioritizedShards != null) && (!prioritizedShards.isEmpty())) { if (wlog.isInfoEnabled()) { StringBuilder builder = new StringBuilder(); boolean firstItem = true; for (ShardInfo shardInfo : prioritizedShards) { if (!firstItem) { builder.append(", "); } builder.append(shardInfo.getShardId()); firstItem = false; } wlog.info("Current stream shard assignments: " + builder.toString()); } } else { wlog.info("No activities assigned"); } return prioritizedShards; }
submittedNewTask = true; LOG.debug("Submitted new " + currentTask.getTaskType() + " task for shard " + shardInfo.getShardId()); } catch (RejectedExecutionException e) { LOG.info(currentTask.getTaskType() + " task was not accepted for execution.", e); if (LOG.isDebugEnabled()) { LOG.debug(String.format("No new task to submit for shard %s, currentState %s", shardInfo.getShardId(), currentState.toString())); final long timeElapsed = System.currentTimeMillis() - currentTaskSubmitTime; final String commonMessage = String.format("Previous %s task still pending for shard %s since %d ms ago. ", currentTask.getTaskType(), shardInfo.getShardId(), timeElapsed); if (LOG.isDebugEnabled()) { LOG.debug(commonMessage + "Not submitting new task.");
this.shard = ((IKinesisProxyExtended) kinesisProxy).getShard(this.shardInfo.getShardId()); } else { this.shard = null;
LOG.debug("Validated checkpoint sequence number " + sequenceNumber + " for " + shardInfo.getShardId() + ", token " + shardInfo.getConcurrencyToken()); LOG.debug("Checkpointing " + shardInfo.getShardId() + ", token " + shardInfo.getConcurrencyToken() + " at specific extended sequence number " + newCheckpoint);
LOG.debug("Validated prepareCheckpoint sequence number " + sequenceNumber + " for " + shardInfo.getShardId() + ", token " + shardInfo.getConcurrencyToken()); LOG.debug("Preparing checkpoint " + shardInfo.getShardId() + ", token " + shardInfo.getConcurrencyToken() + " at specific extended sequence number " + pendingCheckpoint);
+ shardInfo.getShardId()); return new TaskResult(null); Thread.sleep(parentShardPollIntervalMillis); } catch (InterruptedException e) { LOG.error("Sleep interrupted when waiting on parent shard(s) of " + shardInfo.getShardId(), e);
} catch (ExpiredIteratorException e) { LOG.info("ShardId " + shardInfo.getShardId() + ": getRecords threw ExpiredIteratorException - restarting after greatest seqNum " + "passed to customer", e); } catch (ExpiredIteratorException ex) { String msg = "Shard " + shardInfo.getShardId() + ": getRecords threw ExpiredIteratorException with a fresh iterator."; LOG.error(msg, ex);
LOG.debug("Initializing ShardId " + shardInfo.getShardId()); Checkpoint initialCheckpointObject = checkpoint.getCheckpointObject(shardInfo.getShardId()); ExtendedSequenceNumber initialCheckpoint = initialCheckpointObject.getCheckpoint(); .withShardId(shardInfo.getShardId()) .withExtendedSequenceNumber(initialCheckpoint) .withPendingCheckpointSequenceNumber(initialCheckpointObject.getPendingCheckpoint());
try { if (LOG.isDebugEnabled()) { LOG.debug("Setting " + shardInfo.getShardId() + ", token " + shardInfo.getConcurrencyToken() + " checkpoint to " + checkpointToRecord); checkpoint.setCheckpoint(shardInfo.getShardId(), checkpointToRecord, shardInfo.getConcurrencyToken()); lastCheckpointValue = checkpointToRecord; } catch (ThrottlingException | ShutdownException | InvalidStateException
long startTimeMillis = System.currentTimeMillis(); IMetricsScope scope = MetricsHelper.getMetricsScope(); scope.addDimension(MetricsHelper.SHARD_ID_DIMENSION_NAME, shardInfo.getShardId()); scope.addData(RECORDS_PROCESSED_METRIC, 0, StandardUnit.Count, MetricsLevel.SUMMARY); scope.addData(DATA_BYTES_PROCESSED_METRIC, 0, StandardUnit.Bytes, MetricsLevel.SUMMARY); LOG.info("Reached end of shard " + shardInfo.getShardId()); return new TaskResult(null, true); LOG.error("ShardId " + shardInfo.getShardId() + ": Caught exception: ", e); exception = e; backoff();
checkpoint.prepareCheckpoint(shardInfo.getShardId(), newPrepareCheckpoint, shardInfo.getConcurrencyToken()); } catch (ThrottlingException | ShutdownException | InvalidStateException | KinesisClientLibDependencyException e) {