protected ChunkProvider<I> createChunkProvider() { SkipPolicy readSkipPolicy = createSkipPolicy(); readSkipPolicy = getFatalExceptionAwareProxy(readSkipPolicy); FaultTolerantChunkProvider<I> chunkProvider = new FaultTolerantChunkProvider<>(getReader(), createChunkOperations()); chunkProvider.setMaxSkipsOnRead(Math.max(getChunkSize(), FaultTolerantChunkProvider.DEFAULT_MAX_SKIPS_ON_READ)); chunkProvider.setSkipPolicy(readSkipPolicy); chunkProvider.setRollbackClassifier(getRollbackClassifier()); ArrayList<StepListener> listeners = new ArrayList<>(getItemListeners()); listeners.addAll(skipListeners); chunkProvider.setListeners(listeners); return chunkProvider; }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override public void postProcess(StepContribution contribution, Chunk<I> chunk) { for (Exception e : chunk.getErrors()) { try { getListener().onSkipInRead(e); } catch (RuntimeException ex) { throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); } } }
@Test public void testProvideWithOverflow() throws Exception { provider = new FaultTolerantChunkProvider<>(new ItemReader<String>() { @Override public String read() throws Exception, UnexpectedInputException, ParseException { throw new RuntimeException("Planned"); } }, new RepeatTemplate()); provider.setSkipPolicy(new LimitCheckingItemSkipPolicy(Integer.MAX_VALUE, Collections.<Class<? extends Throwable>,Boolean>singletonMap(Exception.class, Boolean.TRUE))); provider.setMaxSkipsOnRead(10); Chunk<String> chunk = null; chunk = provider.provide(contribution); assertNotNull(chunk); assertEquals(0, chunk.getItems().size()); assertEquals(10, chunk.getErrors().size()); } }
@Test public void testProvide() throws Exception { provider = new FaultTolerantChunkProvider<>(new ListItemReader<>(Arrays.asList("foo", "bar")), new RepeatTemplate()); Chunk<String> chunk = provider.provide(contribution); assertNotNull(chunk); assertEquals(2, chunk.getItems().size()); }
protected ChunkProvider<I> createChunkProvider() { SkipPolicy readSkipPolicy = createSkipPolicy(); readSkipPolicy = getFatalExceptionAwareProxy(readSkipPolicy); FaultTolerantChunkProvider<I> chunkProvider = new FaultTolerantChunkProvider<>(getReader(), createChunkOperations()); chunkProvider.setMaxSkipsOnRead(Math.max(getChunkSize(), FaultTolerantChunkProvider.DEFAULT_MAX_SKIPS_ON_READ)); chunkProvider.setSkipPolicy(readSkipPolicy); chunkProvider.setRollbackClassifier(getRollbackClassifier()); ArrayList<StepListener> listeners = new ArrayList<>(getItemListeners()); listeners.addAll(skipListeners); chunkProvider.setListeners(listeners); return chunkProvider; }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override public void postProcess(StepContribution contribution, Chunk<I> chunk) { for (Exception e : chunk.getErrors()) { try { getListener().onSkipInRead(e); } catch (RuntimeException ex) { throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); } } }
protected ChunkProvider<I> createChunkProvider() { SkipPolicy readSkipPolicy = createSkipPolicy(); readSkipPolicy = getFatalExceptionAwareProxy(readSkipPolicy); FaultTolerantChunkProvider<I> chunkProvider = new FaultTolerantChunkProvider<>(getReader(), createChunkOperations()); chunkProvider.setMaxSkipsOnRead(Math.max(getChunkSize(), FaultTolerantChunkProvider.DEFAULT_MAX_SKIPS_ON_READ)); chunkProvider.setSkipPolicy(readSkipPolicy); chunkProvider.setRollbackClassifier(getRollbackClassifier()); ArrayList<StepListener> listeners = new ArrayList<>(getItemListeners()); listeners.addAll(skipListeners); chunkProvider.setListeners(listeners); return chunkProvider; }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override public void postProcess(StepContribution contribution, Chunk<I> chunk) { for (Exception e : chunk.getErrors()) { try { getListener().onSkipInRead(e); } catch (RuntimeException ex) { throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); } } }
protected ChunkProvider<I> createChunkProvider() { SkipPolicy readSkipPolicy = createSkipPolicy(); readSkipPolicy = getFatalExceptionAwareProxy(readSkipPolicy); FaultTolerantChunkProvider<I> chunkProvider = new FaultTolerantChunkProvider<I>(getReader(), createChunkOperations()); chunkProvider.setMaxSkipsOnRead(Math.max(getChunkSize(), FaultTolerantChunkProvider.DEFAULT_MAX_SKIPS_ON_READ)); chunkProvider.setSkipPolicy(readSkipPolicy); chunkProvider.setRollbackClassifier(getRollbackClassifier()); ArrayList<StepListener> listeners = new ArrayList<StepListener>(getItemListeners()); listeners.addAll(skipListeners); chunkProvider.setListeners(listeners); return chunkProvider; }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override public void postProcess(StepContribution contribution, Chunk<I> chunk) { for (Exception e : chunk.getErrors()) { try { getListener().onSkipInRead(e); } catch (RuntimeException ex) { throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); } } }
/** * @return {@link ChunkProvider} configured for fault-tolerance. */ @Override protected SimpleChunkProvider<T> configureChunkProvider() { if (skipPolicy != null) { if (!skippableExceptionClasses.isEmpty()) { logger.info("Skippable exceptions will be ignored because a SkipPolicy was specified explicitly"); } if (skipLimit > 0) { logger.info("Skip limit will be ignored because a SkipPolicy was specified explicitly"); } } SkipPolicy readSkipPolicy = skipPolicy != null ? skipPolicy : new LimitCheckingItemSkipPolicy(skipLimit, getSkippableExceptionClasses()); readSkipPolicy = getFatalExceptionAwareProxy(readSkipPolicy); FaultTolerantChunkProvider<T> chunkProvider = new FaultTolerantChunkProvider<T>(getItemReader(), getChunkOperations()); chunkProvider.setMaxSkipsOnRead(Math.max(getCommitInterval(), FaultTolerantChunkProvider.DEFAULT_MAX_SKIPS_ON_READ)); chunkProvider.setSkipPolicy(readSkipPolicy); chunkProvider.setRollbackClassifier(getRollbackClassifier()); return chunkProvider; }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override public void postProcess(StepContribution contribution, Chunk<I> chunk) { for (Exception e : chunk.getErrors()) { try { getListener().onSkipInRead(e); } catch (RuntimeException ex) { throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); } } }