@Override public Chunk<T> provide(StepContribution contribution) throws Exception { return new Chunk<>(); }
@Override public Chunk<I> provide(final StepContribution contribution) throws Exception { final Chunk<I> inputs = new Chunk<>(); repeatOperations.iterate(new RepeatCallback() { @Override public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; } }); return inputs; }
public AlmostStatefulRetryChunkTests(String[] args, int limit) { chunk = new Chunk<>(); for (String string : args) { chunk.add(string); } this.retryLimit = limit; }
@Test public void testWrite() throws Exception { Chunk<String> inputs = new Chunk<>(Arrays.asList("1", "2")); processor.process(contribution, inputs); assertEquals(2, list.size()); }
@Override public Chunk<String> provide(StepContribution contribution) throws Exception { contribution.incrementReadCount(); Chunk<String> chunk = new Chunk<>(); chunk.add("foo"); return chunk; } @Override
@Override protected void initializeUserData(Chunk<I> inputs) { @SuppressWarnings("unchecked") UserData<O> data = (UserData<O>) inputs.getUserData(); if (data == null) { data = new UserData<>(); inputs.setUserData(data); data.setOutputs(new Chunk<>()); } else { // BATCH-2663: re-initialize filter count when scanning the chunk if (data.scanning()) { data.filterCount = 0; } } }
@Override public Chunk<String> provide(StepContribution contribution) throws Exception { contribution.incrementReadCount(); Chunk<String> chunk = new Chunk<>(); chunk.add("foo"); chunk.setEnd(); return chunk; } @Override
@Override protected Chunk<O> getAdjustedOutputs(Chunk<I> inputs, Chunk<O> outputs) { @SuppressWarnings("unchecked") UserData<O> data = (UserData<O>) inputs.getUserData(); Chunk<O> previous = data.getOutputs(); Chunk<O> next = new Chunk<>(outputs.getItems(), previous.getSkips()); next.setBusy(previous.isBusy()); // Remember for next time if there are skips accumulating data.setOutputs(next); return next; }
@Test public void testTransform() throws Exception { processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { return item.equals("1") ? null : item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("1", "2")); processor.process(contribution, inputs); assertEquals(1, list.size()); assertEquals(1, contribution.getFilterCount()); }
@Test public void testOnErrorInWrite() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "fail")); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void onWriteError(Exception e, List<? extends String> item) { writeError.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk);// Process foo, fail processor.process(contribution, chunk);// Process foo processAndExpectPlannedRuntimeException(chunk);// Process fail assertEquals("[foo, fail, fail]", writeError.toString()); }
protected Chunk<O> transform(StepContribution contribution, Chunk<I> inputs) throws Exception { Chunk<O> outputs = new Chunk<>(); for (Chunk<I>.ChunkIterator iterator = inputs.iterator(); iterator.hasNext();) { final I item = iterator.next(); O output; try { output = doProcess(item); } catch (Exception e) { /* * For a simple chunk processor (no fault tolerance) we are done * here, so prevent any more processing of these inputs. */ inputs.clear(); throw e; } if (output != null) { outputs.add(output); } else { iterator.remove(); } } return outputs; }
@Test public void testTransformWithExceptionAndNoRollback() throws Exception { processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { throw new DataIntegrityViolationException("Planned"); } return item; } }); processor.setProcessSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor .setRollbackClassifier(new BinaryExceptionClassifier( Collections .<Class<? extends Throwable>> singleton(DataIntegrityViolationException.class), false)); Chunk<String> inputs = new Chunk<>(Arrays.asList("1", "2")); processor.process(contribution, inputs); assertEquals(1, list.size()); }
/** * An Error can be retried or skipped but by default it is just propagated * * @throws Exception */ @Test public void testWriteSkipOnError() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { if (items.contains("fail")) { assertFalse("Expected Error!", true); } } }); Chunk<String> inputs = new Chunk<>( Arrays.asList("3", "fail", "2")); try { processor.process(contribution, inputs); fail("Expected Error"); } catch (Error e) { assertEquals("Expected Error!", e.getMessage()); } processor.process(contribution, inputs); }
@Test public void testAfterWrite() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "fail", "bar")); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void afterWrite(List<? extends String> item) { after.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk); processor.process(contribution, chunk); assertEquals(2, chunk.getItems().size()); processAndExpectPlannedRuntimeException(chunk); assertEquals(1, chunk.getItems().size()); processor.process(contribution, chunk); assertEquals(0, chunk.getItems().size()); // foo is written once because it the failure is detected before it is // committed the first time assertEquals("[foo, bar]", list.toString()); // the after listener is called once per successful item, which is // important assertEquals("[foo, bar]", after.toString()); }
@Test public void testProcess() throws Exception { Chunk<String> chunk = new Chunk<>(); chunk.add("foo"); chunk.add("err"); chunk.add("bar"); processor.process(contribution, chunk); assertEquals(Arrays.asList("foo", "bar"), list); assertEquals(1, contribution.getFilterCount()); assertEquals(2, contribution.getWriteCount()); }
.<Class<? extends Throwable>> singleton(IllegalArgumentException.class), false)); processor.afterPropertiesSet(); Chunk<String> inputs = new Chunk<>(Arrays.asList("1", "2", "skip", "skip", "3", "fail", "fail", "4", "5")); processor.process(contribution, inputs); assertEquals(5, list.size());
@Test public void testOnErrorInWriteAllItemsFail() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "bar")); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { // Always fail in writer throw new RuntimeException("Planned failure!"); } }, batchRetryTemplate); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void onWriteError(Exception e, List<? extends String> item) { writeError.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk);// Process foo, bar processAndExpectPlannedRuntimeException(chunk);// Process foo processAndExpectPlannedRuntimeException(chunk);// Process bar assertEquals("[foo, bar, foo, bar]", writeError.toString()); }
@Test // BATCH-2663 public void testFilterCountOnSkipInWriteWithoutRetry() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { return null; } return item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("fail", "1", "2")); processAndExpectPlannedRuntimeException(inputs); // (first attempt) Process fail, 1, 2 // item 1 is filtered out so it is removed from the chunk => now inputs = [fail, 2] // using NeverRetryPolicy by default => now scanning processAndExpectPlannedRuntimeException(inputs); // (scanning) Process fail processor.process(contribution, inputs); // (scanning) Process 2 assertEquals(1, list.size()); assertEquals("[2]", list.toString()); assertEquals(1, contribution.getWriteSkipCount()); assertEquals(1, contribution.getFilterCount()); }
@Test public void testFilterCountOnSkip() throws Exception { processor.setProcessSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { throw new RuntimeException("Skippable"); } if (item.equals("3")) { return null; } return item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("3", "1", "2")); try { processor.process(contribution, inputs); fail("Expected Exception"); } catch (Exception e) { assertEquals("Skippable", e.getMessage()); } processor.process(contribution, inputs); assertEquals(1, list.size()); assertEquals(1, contribution.getSkipCount()); assertEquals(1, contribution.getFilterCount()); }
Chunk<String> inputs = new Chunk<>(Arrays.asList("fail")); try { processor.process(contribution, inputs);