/** * @return a timestamp which determines partition expiry. Partitions with a timestamp smaller (older) than this value * are considered 'expired'. */ protected long getExpiryBorder() { long now = System.currentTimeMillis(); long expirationTimeoutMillis = TimeUnit.SECONDS.toMillis(getConfiguration().getTimeout()); return now - expirationTimeoutMillis; }
/** * Resets the process state of the given partition keys, as they were not successfully processed, or discards the * partition if it has already been attempted the configured number of attempts. */ protected void abort(ConsumerWorkingSet workingSet, List<? extends PartitionKey> partitionKeys) { List<PartitionKey> discardedPartitions = new ArrayList<>(); for (PartitionKey key : partitionKeys) { ConsumablePartition consumablePartition = workingSet.lookup(key); assertInProgress(consumablePartition); // either reset its processState, or remove it from the workingSet, depending on how many tries it already has if (consumablePartition.getNumFailures() < getConfiguration().getMaxRetries()) { consumablePartition.retry(); } else { discardedPartitions.add(key); workingSet.lookup(key).discard(); } } if (!discardedPartitions.isEmpty()) { LOG.warn("Discarded keys due to being retried {} times: {}", getConfiguration().getMaxRetries(), discardedPartitions); } }
/** * Goes through all partitions. If any IN_PROGRESS partition is older than the configured timeout, reset its state * to AVAILABLE, unless it has already been retried the configured number of times, in which case it is discarded. */ protected void doExpiry(ConsumerWorkingSet workingSet) { long expiryTime = getExpiryBorder(); List<PartitionKey> expiredPartitions = new ArrayList<>(); List<PartitionKey> discardedPartitions = new ArrayList<>(); for (ConsumablePartition partition : workingSet.getPartitions()) { if (partition.getProcessState() == ProcessState.IN_PROGRESS && partition.getTimestamp() < expiryTime) { // either reset its processState, or remove it from the workingSet, depending on how many tries it already has if (partition.getNumFailures() < getConfiguration().getMaxRetries()) { partition.retry(); } else { partition.discard(); } expiredPartitions.add(partition.getPartitionKey()); } } if (!expiredPartitions.isEmpty()) { LOG.warn("Expiring in progress partitions: {}", expiredPartitions); if (!discardedPartitions.isEmpty()) { LOG.warn("Discarded keys due to being retried {} times: {}", getConfiguration().getMaxRetries(), discardedPartitions); } } } }
@Override public PartitionConsumerResult doConsume(ConsumerWorkingSet workingSet, PartitionAcceptor acceptor) { doExpiry(workingSet); workingSet.populate(getPartitionedFileSet(), getConfiguration()); List<PartitionDetail> toConsume = selectPartitions(acceptor, workingSet); return new PartitionConsumerResult(toConsume, removeDiscardedPartitions(workingSet)); }