@Override public void beginTransaction() { Notifications.notifyBeginTransaction(morphline); }
@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
@Override public void commitTransaction() { Notifications.notifyCommitTransaction(morphline); }
@Test public void testNotifications() throws Exception { morphline = createMorphline("test-morphlines/pipeWithTwoBasicCommands"); Notifications.notifyBeginTransaction(morphline); Notifications.notifyStartSession(morphline); Notifications.notifyCommitTransaction(morphline); Notifications.notifyRollbackTransaction(morphline); }
/** * Determine whether or not the given notification contains the given lifecycle event. * @param notification A {@link Record} that represents a notification. * @param event A {@link LifecycleEvent} enumeration that is to be searched for in the given notification. */ public static boolean containsLifecycleEvent(Record notification, LifecycleEvent event) { return getLifecycleEvents(notification).contains(event); }
public static List<Record> executePipeline(Pipeline pipeline, Record inputRecord, boolean errorOnEmpty) { Command morphline = pipeline.getMorphline(); try { LOG.trace("Input Record: {}", inputRecord); // Process the Record Notifications.notifyStartSession(morphline); boolean success = morphline.process(inputRecord); Notifications.notifyCommitTransaction(morphline); if (!success) { throw new MorphlineRuntimeException("Morphline failed to process incoming Record: " + inputRecord); } // Collect the output List<Record> outputRecords = pipeline.getCollector().getRecords(); if (errorOnEmpty && !outputRecords.iterator().hasNext()) { throw new MorphlineRuntimeException("Morphline did not produce output Record(s)"); } LOG.trace("Output Record(s): {}", outputRecords); return outputRecords; } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); // TODO : Review exception handling LOG.warn("Morphline failed to execute properly on incoming Record: " + inputRecord, e); throw e; } }
public void cleanup() { Notifications.notifyCommitTransaction(morphline); Notifications.notifyShutdown(morphline); }
record.put("user_friends_count", 123); record.put("text", "myText"); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getRecords().size()); assertEquals(Arrays.asList("hello sun", "goodbye mars"), docs.get(0).get("text")); Notifications.notifyCommitTransaction(morphline); Notifications.notifyShutdown(morphline);
@Test public void testTokenizeText() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "tokenizeText"); for (int i = 0; i < 3; i++) { Record record = new Record(); record.put(Fields.MESSAGE, "Hello World!"); record.put(Fields.MESSAGE, "\nFoo@Bar.com #%()123"); Record expected = record.copy(); expected.getFields().putAll("tokens", Arrays.asList("hello", "world", "foo", "bar.com", "123")); collector.reset(); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); assertEquals(expected, collector.getFirstRecord()); } }
record.replaceValues(Fields.ID, "id0"); record.replaceValues("first_name", "Nadja"); // will be sanitized Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record.copy())); assertEquals(1, query("*:*").getResults().size()); assertNull(docs.get(1).getFirstValue(LoadSolrBuilder.LOAD_SOLR_DELETE_BY_QUERY)); Notifications.notifyRollbackTransaction(morphline); Notifications.notifyShutdown(morphline);
@Override public void stop() { Notifications.notifyShutdown(morphline); }
@Override protected void doNotify(Record notification) { if (Notifications.containsLifecycleEvent(notification, Notifications.LifecycleEvent.START_SESSION)) { recordCounter = 0; // reset } super.doNotify(notification); }
@Override public void rollbackTransaction() { Notifications.notifyRollbackTransaction(morphline); }
/** Usage: java ... <morphline.conf> <dataFile1> ... <dataFileN> */ public static void main(String[] args) throws IOException { // compile morphline.conf file on the fly File morphlineFile = new File(args[0]); String morphlineId = null; MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null); // process each input data file Notifications.notifyBeginTransaction(morphline); try { for (int i = 1; i < args.length; i++) { InputStream in = new BufferedInputStream(new FileInputStream(new File(args[i]))); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); Notifications.notifyStartSession(morphline); boolean success = morphline.process(record); if (!success) { System.out.println("Morphline failed to process record: " + record); } in.close(); } Notifications.notifyCommitTransaction(morphline); } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); morphlineContext.getExceptionHandler().handleException(e, null); } Notifications.notifyShutdown(morphline); } }
/** * Determine whether or not the given notification contains the given lifecycle event. * @param notification A {@link Record} that represents a notification. * @param event A {@link LifecycleEvent} enumeration that is to be searched for in the given notification. */ public static boolean containsLifecycleEvent(Record notification, LifecycleEvent event) { return getLifecycleEvents(notification).contains(event); }
public void cleanup() { Notifications.notifyCommitTransaction(morphline); Notifications.notifyShutdown(morphline); }
@Test public void testLoadSolrWithChildDocuments() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "loadSolrWithChildDocuments"); Record record = new Record(); record.put(Fields.ID, "id0"); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); // This parent block join returns the parent records for records // where the child documents contain "bar" in the id field. SolrDocumentList docs = query("{!parent which='content_type:parent'}id:bar").getResults(); assertEquals(1, docs.size()); assertEquals("id0", docs.get(0).getFirstValue(Fields.ID)); docs = query("*:*").getResults(); assertEquals(3, docs.size()); }
@Override public void stop() { Notifications.notifyShutdown(morphline); }
@Override public void notify(Record notification) { if (Notifications.containsLifecycleEvent(notification, Notifications.LifecycleEvent.START_SESSION)) { numStartEvents++; } }
@Override public void rollbackTransaction() { Notifications.notifyRollbackTransaction(morphline); }