/** * Resets the process state of the given partition keys, as they were not successfully processed, or discards the * partition if it has already been attempted the configured number of attempts. */ protected void abort(ConsumerWorkingSet workingSet, List<? extends PartitionKey> partitionKeys) { List<PartitionKey> discardedPartitions = new ArrayList<>(); for (PartitionKey key : partitionKeys) { ConsumablePartition consumablePartition = workingSet.lookup(key); assertInProgress(consumablePartition); // either reset its processState, or remove it from the workingSet, depending on how many tries it already has if (consumablePartition.getNumFailures() < getConfiguration().getMaxRetries()) { consumablePartition.retry(); } else { discardedPartitions.add(key); workingSet.lookup(key).discard(); } } if (!discardedPartitions.isEmpty()) { LOG.warn("Discarded keys due to being retried {} times: {}", getConfiguration().getMaxRetries(), discardedPartitions); } }
/** * Create a ConsumerConfiguration from this builder, using the private ConsumerConfiguration * constructor. */ public ConsumerConfiguration build() { return new ConsumerConfiguration(partitionPredicate, maxWorkingSetSize, timeout, maxRetries); } }
/** * Populates the ConsumerWorkingSet by fetching partitions from the given PartitionedFileSet. * * @param partitionedFileSet the PartitionedFileSet to fetch partitions from * @param configuration the ConsumerConfiguration which defines parameters for consuming */ public void populate(PartitionedFileSet partitionedFileSet, ConsumerConfiguration configuration) { int numToPopulate = configuration.getMaxWorkingSetSize() - partitions.size(); Predicate<PartitionDetail> predicate = configuration.getPartitionPredicate(); co.cask.cdap.api.dataset.lib.PartitionConsumerResult result = partitionedFileSet.consumePartitions(partitionConsumerState, numToPopulate, predicate); List<PartitionDetail> partitions = result.getPartitions(); for (PartitionDetail partition : partitions) { addPartition(partition.getPartitionKey()); } partitionConsumerState = result.getPartitionConsumerState(); }
ConsumerConfiguration.builder().setPartitionPredicate(predicate).build(); PartitionConsumer newPartitionConsumer = new ConcurrentPartitionConsumer(dataset, new InMemoryStatePersistor(), configuration);
/** * @return a timestamp which determines partition expiry. Partitions with a timestamp smaller (older) than this value * are considered 'expired'. */ protected long getExpiryBorder() { long now = System.currentTimeMillis(); long expirationTimeoutMillis = TimeUnit.SECONDS.toMillis(getConfiguration().getTimeout()); return now - expirationTimeoutMillis; }
ConsumerConfiguration.builder() .setMaxRetries(3) .build();
/** * Goes through all partitions. If any IN_PROGRESS partition is older than the configured timeout, reset its state * to AVAILABLE, unless it has already been retried the configured number of times, in which case it is discarded. */ protected void doExpiry(ConsumerWorkingSet workingSet) { long expiryTime = getExpiryBorder(); List<PartitionKey> expiredPartitions = new ArrayList<>(); List<PartitionKey> discardedPartitions = new ArrayList<>(); for (ConsumablePartition partition : workingSet.getPartitions()) { if (partition.getProcessState() == ProcessState.IN_PROGRESS && partition.getTimestamp() < expiryTime) { // either reset its processState, or remove it from the workingSet, depending on how many tries it already has if (partition.getNumFailures() < getConfiguration().getMaxRetries()) { partition.retry(); } else { partition.discard(); } expiredPartitions.add(partition.getPartitionKey()); } } if (!expiredPartitions.isEmpty()) { LOG.warn("Expiring in progress partitions: {}", expiredPartitions); if (!discardedPartitions.isEmpty()) { LOG.warn("Discarded keys due to being retried {} times: {}", getConfiguration().getMaxRetries(), discardedPartitions); } } } }
ConsumerConfiguration.builder().setMaxRetries(1).build()); dsFrameworkUtil.newInMemoryTransactionExecutor(txAwareDataset).execute(new TransactionExecutor.Subroutine() { @Override
private boolean isLastAttempt(ConsumablePartition partition) { return partition.getNumFailures() == getConfiguration().getMaxRetries() - 1; } }
ConsumerConfiguration configuration = ConsumerConfiguration.builder().setMaxRetries(3).build();
ConsumerConfiguration consumerConfiguration = ConsumerConfiguration.builder().setMaxWorkingSetSize(100).build();
ConsumerConfiguration configuration = ConsumerConfiguration.builder().setMaxRetries(numRetries).build(); final PartitionConsumer partitionConsumer = new ConcurrentPartitionConsumer(dataset, new InMemoryStatePersistor(), configuration);
final TransactionAware txAwareDataset = (TransactionAware) dataset; ConsumerConfiguration configuration = ConsumerConfiguration.builder().setMaxRetries(3).build(); final PartitionConsumer partitionConsumer = new ConcurrentPartitionConsumer(dataset, new InMemoryStatePersistor(), configuration);
final TransactionAware txAwareDataset = (TransactionAware) dataset; ConsumerConfiguration configuration = ConsumerConfiguration.builder() .setMaxWorkingSetSize(1)