/** * Goes through all partitions. If any IN_PROGRESS partition is older than the configured timeout, reset its state * to AVAILABLE, unless it has already been retried the configured number of times, in which case it is discarded. */ protected void doExpiry(ConsumerWorkingSet workingSet) { long expiryTime = getExpiryBorder(); List<PartitionKey> expiredPartitions = new ArrayList<>(); List<PartitionKey> discardedPartitions = new ArrayList<>(); for (ConsumablePartition partition : workingSet.getPartitions()) { if (partition.getProcessState() == ProcessState.IN_PROGRESS && partition.getTimestamp() < expiryTime) { // either reset its processState, or remove it from the workingSet, depending on how many tries it already has if (partition.getNumFailures() < getConfiguration().getMaxRetries()) { partition.retry(); } else { partition.discard(); } expiredPartitions.add(partition.getPartitionKey()); } } if (!expiredPartitions.isEmpty()) { LOG.warn("Expiring in progress partitions: {}", expiredPartitions); if (!discardedPartitions.isEmpty()) { LOG.warn("Discarded keys due to being retried {} times: {}", getConfiguration().getMaxRetries(), discardedPartitions); } } } }
private List<PartitionDetail> selectPartitions(PartitionAcceptor acceptor, ConsumerWorkingSet workingSet) { long now = System.currentTimeMillis(); List<PartitionDetail> toConsume = new ArrayList<>(); Iterator<ConsumablePartition> iter = workingSet.getPartitions().iterator(); while (iter.hasNext()) { ConsumablePartition consumablePartition = iter.next(); if (ProcessState.AVAILABLE != consumablePartition.getProcessState()) { continue; } PartitionDetail partition = getPartitionedFileSet().getPartition(consumablePartition.getPartitionKey()); if (partition == null) { // no longer exists, so skip it and remove it from the working set iter.remove(); continue; } PartitionAcceptor.Return accept = acceptor.accept(partition); switch (accept) { case ACCEPT: consumablePartition.take(); consumablePartition.setTimestamp(now); toConsume.add(partition); continue; case SKIP: continue; case STOP: return toConsume; } } return toConsume; }
@Override public void untake(ConsumerWorkingSet workingSet, List<? extends PartitionKey> partitionKeys) { doExpiry(workingSet); for (PartitionKey key : partitionKeys) { ConsumablePartition consumablePartition = workingSet.lookup(key); // don't need to assertInProgress because untake() already does that consumablePartition.untake(); } }
/** * ensure that caller doesn't try to commit/abort a partition that isn't in progress * @throws IllegalArgumentException if the given partition is in progress */ protected void assertInProgress(ConsumablePartition consumablePartition) { if (!(consumablePartition.getProcessState() == ProcessState.IN_PROGRESS)) { throw new IllegalStateException(String.format("Partition not in progress: %s", consumablePartition.getPartitionKey())); } }
@Test public void testSimpleTransitions() { // tests simple success case ConsumablePartition partition = new DefaultConsumablePartition(generateUniqueKey()); Assert.assertEquals(0, partition.getNumFailures()); partition.take(); partition.retry(); Assert.assertEquals(1, partition.getNumFailures()); partition.take(); // test that untake doesn't increment failure count partition.untake(); Assert.assertEquals(1, partition.getNumFailures()); partition.take(); partition.complete(); }
/** * Resets the process state of the given partition keys, as they were not successfully processed, or discards the * partition if it has already been attempted the configured number of attempts. */ protected void abort(ConsumerWorkingSet workingSet, List<? extends PartitionKey> partitionKeys) { List<PartitionKey> discardedPartitions = new ArrayList<>(); for (PartitionKey key : partitionKeys) { ConsumablePartition consumablePartition = workingSet.lookup(key); assertInProgress(consumablePartition); // either reset its processState, or remove it from the workingSet, depending on how many tries it already has if (consumablePartition.getNumFailures() < getConfiguration().getMaxRetries()) { consumablePartition.retry(); } else { discardedPartitions.add(key); workingSet.lookup(key).discard(); } } if (!discardedPartitions.isEmpty()) { LOG.warn("Discarded keys due to being retried {} times: {}", getConfiguration().getMaxRetries(), discardedPartitions); } }
@Test public void testByteSerialization() { ConsumerWorkingSet workingSet = new ConsumerWorkingSet(); // test with empty partitions lists testSerDe(workingSet); // test with two elements in AVAILABLE and none in IN_PROGRESS workingSet.getPartitions().add(new DefaultConsumablePartition(generateUniqueKey())); workingSet.getPartitions().add(new DefaultConsumablePartition(generateUniqueKey())); testSerDe(workingSet); // test with three elements in partitions and none in inProgressPartitions workingSet.getPartitions().add(new DefaultConsumablePartition(generateUniqueKey())); testSerDe(workingSet); // mark the first element as IN_PROGRESS workingSet.getPartitions().get(0).take(); workingSet.getPartitions().get(0).setTimestamp(System.currentTimeMillis()); testSerDe(workingSet); }
@Test(expected = IllegalStateException.class) public void testAlreadyCompletedTransition() { // cannot complete a partition that has already been completed ConsumablePartition partition = new DefaultConsumablePartition(generateUniqueKey()); partition.take(); partition.complete(); partition.complete(); }
/** * @return the ConsumablePartition with the given PartitionKey, from the working set, after removing it from * the partitions list */ public ConsumablePartition remove(PartitionKey partitionKey) { for (int i = 0; i < partitions.size(); i++) { if (partitionKey.equals(partitions.get(i).getPartitionKey())) { return partitions.remove(i); } } throw new IllegalArgumentException("PartitionKey not found: " + partitionKey); }
private boolean isLastAttempt(ConsumablePartition partition) { return partition.getNumFailures() == getConfiguration().getMaxRetries() - 1; } }
@Test(expected = IllegalStateException.class) public void testRetryWithoutTakenTransition() { // cannot take retry a partition without it being taken first ConsumablePartition partition = new DefaultConsumablePartition(generateUniqueKey()); partition.retry(); }
/** * Removes the list of partitions that have failed processing the configured number of times from the working set and * returns them. */ protected List<PartitionDetail> removeDiscardedPartitions(ConsumerWorkingSet workingSet) { List<PartitionDetail> failedPartitions = new ArrayList<>(); Iterator<ConsumablePartition> iter = workingSet.getPartitions().iterator(); while (iter.hasNext()) { ConsumablePartition partition = iter.next(); if (partition.getProcessState() == ProcessState.DISCARDED) { failedPartitions.add(getPartitionedFileSet().getPartition(partition.getPartitionKey())); iter.remove(); } } return failedPartitions; }
/** * @return the ConsumablePartition with the given PartitionKey, from the working set */ public ConsumablePartition lookup(PartitionKey partitionKey) { for (int i = 0; i < partitions.size(); i++) { if (partitionKey.equals(partitions.get(i).getPartitionKey())) { return partitions.get(i); } } throw new IllegalArgumentException("PartitionKey not found: " + partitionKey); }
ConsumablePartition firstPartition = partitions.get(0); if (isLastAttempt(firstPartition)) { firstPartition.take(); firstPartition.setTimestamp(now); toConsume.add(getPartitionedFileSet().getPartition(firstPartition.getPartitionKey())); return new PartitionConsumerResult(toConsume, removeDiscardedPartitions(workingSet)); if (ProcessState.AVAILABLE != consumablePartition.getProcessState()) { continue; continue; PartitionDetail partition = getPartitionedFileSet().getPartition(consumablePartition.getPartitionKey()); PartitionAcceptor.Return accept = acceptor.accept(partition); switch (accept) { case ACCEPT: consumablePartition.take(); consumablePartition.setTimestamp(now); toConsume.add(partition); continue;