@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
protected void startSession() { Notifications.notifyStartSession(morphline); }
protected void startSession() { Notifications.notifyStartSession(morphline); }
protected void startSession() { Notifications.notifyStartSession(morphline); }
private boolean load(Record record) { Notifications.notifyStartSession(morphline); return morphline.process(record); }
@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
@Override public void map(Result result, SolrUpdateWriter solrUpdateWriter) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, result); record.put(Fields.ATTACHMENT_MIME_TYPE, MorphlineResultToSolrMapper.OUTPUT_MIME_TYPE); for (Map.Entry<String, String> entry : forcedRecordFields.entrySet()) { record.replaceValues(entry.getKey(), entry.getValue()); } collector.reset(solrUpdateWriter); try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { collector.reset(null); timerContext.stop(); } }
@Override public void map(Result result, SolrUpdateWriter solrUpdateWriter) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, result); record.put(Fields.ATTACHMENT_MIME_TYPE, MorphlineResultToSolrMapper.OUTPUT_MIME_TYPE); for (Map.Entry<String, String> entry : forcedRecordFields.entrySet()) { record.replaceValues(entry.getKey(), entry.getValue()); } collector.reset(solrUpdateWriter); try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { collector.reset(null); timerContext.stop(); } }
public static List<Record> executePipeline(Pipeline pipeline, Record inputRecord, boolean errorOnEmpty) { Command morphline = pipeline.getMorphline(); try { LOG.trace("Input Record: {}", inputRecord); // Process the Record Notifications.notifyStartSession(morphline); boolean success = morphline.process(inputRecord); Notifications.notifyCommitTransaction(morphline); if (!success) { throw new MorphlineRuntimeException("Morphline failed to process incoming Record: " + inputRecord); } // Collect the output List<Record> outputRecords = pipeline.getCollector().getRecords(); if (errorOnEmpty && !outputRecords.iterator().hasNext()) { throw new MorphlineRuntimeException("Morphline did not produce output Record(s)"); } LOG.trace("Output Record(s): {}", outputRecords); return outputRecords; } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); // TODO : Review exception handling LOG.warn("Morphline failed to execute properly on incoming Record: " + inputRecord, e); throw e; } }
Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record);
Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record);
Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark();
private void processAndVerifySuccess(Command myMorphline, Collector myCollector, Record input, Multimap... expectedMaps) { myCollector.reset(); Notifications.notifyStartSession(myMorphline); assertEquals(1, myCollector.getNumStartEvents()); assertTrue(myMorphline.process(input)); Iterator<Record> iter = myCollector.getRecords().iterator(); int i = 0; for (Multimap expected : expectedMaps) { //System.out.println("iter" + i); assertTrue(iter.hasNext()); assertEquals(expected, iter.next().getFields()); i++; } assertFalse(iter.hasNext()); }
/** Usage: java ... <morphline.conf> <dataFile1> ... <dataFileN> */ public static void main(String[] args) throws IOException { // compile morphline.conf file on the fly File morphlineFile = new File(args[0]); String morphlineId = null; MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null); // process each input data file Notifications.notifyBeginTransaction(morphline); try { for (int i = 1; i < args.length; i++) { InputStream in = new BufferedInputStream(new FileInputStream(new File(args[i]))); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); Notifications.notifyStartSession(morphline); boolean success = morphline.process(record); if (!success) { System.out.println("Morphline failed to process record: " + record); } in.close(); } Notifications.notifyCommitTransaction(morphline); } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); morphlineContext.getExceptionHandler().handleException(e, null); } Notifications.notifyShutdown(morphline); } }
@Test public void testNotifications() throws Exception { morphline = createMorphline("test-morphlines/pipeWithTwoBasicCommands"); Notifications.notifyBeginTransaction(morphline); Notifications.notifyStartSession(morphline); Notifications.notifyCommitTransaction(morphline); Notifications.notifyRollbackTransaction(morphline); }