@Override public void afterJobEnd(final JobReport jobReport) { try { blockingQueueRecordWriter.writeRecords(new Batch(new PoisonRecord())); } catch (Exception e) { throw new RuntimeException("Unable to write poison record.", e); } }
private void writeBatch(Batch batch) throws BatchWritingException { LOGGER.log(Level.FINE, "Writing {0}", batch); try { if (!batch.isEmpty()) { recordWriterListener.beforeRecordWriting(batch); recordWriter.writeRecords(batch); recordWriterListener.afterRecordWriting(batch); batchListener.afterBatchWriting(batch); metrics.incrementWriteCount(batch.size()); } } catch (Exception e) { recordWriterListener.onRecordWritingException(batch, e); batchListener.onBatchWritingException(batch, e); throw new BatchWritingException("Unable to write records", e); } }
@SuppressWarnings(value = "unchecked") private void processRecord(Record record, Batch batch) throws ErrorThresholdExceededException { Record processedRecord = null; try { LOGGER.log(Level.FINE, "Processing {0}", record); notifyJobUpdate(); Record preProcessedRecord = pipelineListener.beforeRecordProcessing(record); if (preProcessedRecord == null) { LOGGER.log(Level.FINE, "{0} has been filtered", record); metrics.incrementFilterCount(); } else { processedRecord = recordProcessor.processRecord(preProcessedRecord); if (processedRecord == null) { LOGGER.log(Level.FINE, "{0} has been filtered", record); metrics.incrementFilterCount(); } else { batch.addRecord(processedRecord); } } pipelineListener.afterRecordProcessing(record, processedRecord); } catch (Exception e) { LOGGER.log(Level.SEVERE, "Unable to process " + record, e); pipelineListener.onRecordProcessingException(record, e); metrics.incrementErrorCount(); report.setLastError(e); if (metrics.getErrorCount() > parameters.getErrorThreshold()) { throw new ErrorThresholdExceededException("Error threshold exceeded. Aborting execution", e); } } }
@Override public void afterJobEnd(final JobReport jobReport) { try { broadcastJmsQueueRecordWriter.writeRecords(new Batch(new JmsPoisonRecord(new Header(0L, "Poison record", new Date()), new JmsPoisonMessage()))); } catch (Exception e) { throw new RuntimeException("Unable to broadcast poison record.", e); } }
private Batch readAndProcessBatch() throws RecordReadingException, ErrorThresholdExceededException { Batch batch = new Batch(); batchListener.beforeBatchReading(); for (int i = 0; i < parameters.getBatchSize(); i++) { Record record = readRecord(); if (record == null) { recordTracker.noMoreRecords(); break; } else { metrics.incrementReadCount(); } processRecord(record, batch); } batchListener.afterBatchProcessing(batch); return batch; }