@Override public void commitTransaction() { Notifications.notifyCommitTransaction(morphline); }
@Override public void commitTransaction() { Notifications.notifyCommitTransaction(morphline); }
public void cleanup() { Notifications.notifyCommitTransaction(morphline); Notifications.notifyShutdown(morphline); }
protected void commit() throws Exception { Notifications.notifyCommitTransaction(morphline); }
public void cleanup() { Notifications.notifyCommitTransaction(morphline); Notifications.notifyShutdown(morphline); }
@Override public void cleanup(Emitter<T> emitter) { try { collector.setEmitter(emitter); Notifications.notifyCommitTransaction(morphline); Notifications.notifyShutdown(morphline); } finally { addMetricsToMRCounters(morphlineContext.getMetricRegistry()); } }
public static List<Record> executePipeline(Pipeline pipeline, Record inputRecord, boolean errorOnEmpty) { Command morphline = pipeline.getMorphline(); try { LOG.trace("Input Record: {}", inputRecord); // Process the Record Notifications.notifyStartSession(morphline); boolean success = morphline.process(inputRecord); Notifications.notifyCommitTransaction(morphline); if (!success) { throw new MorphlineRuntimeException("Morphline failed to process incoming Record: " + inputRecord); } // Collect the output List<Record> outputRecords = pipeline.getCollector().getRecords(); if (errorOnEmpty && !outputRecords.iterator().hasNext()) { throw new MorphlineRuntimeException("Morphline did not produce output Record(s)"); } LOG.trace("Output Record(s): {}", outputRecords); return outputRecords; } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); // TODO : Review exception handling LOG.warn("Morphline failed to execute properly on incoming Record: " + inputRecord, e); throw e; } }
/** Usage: java ... <morphline.conf> <dataFile1> ... <dataFileN> */ public static void main(String[] args) throws IOException { // compile morphline.conf file on the fly File morphlineFile = new File(args[0]); String morphlineId = null; MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null); // process each input data file Notifications.notifyBeginTransaction(morphline); try { for (int i = 1; i < args.length; i++) { InputStream in = new BufferedInputStream(new FileInputStream(new File(args[i]))); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); Notifications.notifyStartSession(morphline); boolean success = morphline.process(record); if (!success) { System.out.println("Morphline failed to process record: " + record); } in.close(); } Notifications.notifyCommitTransaction(morphline); } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); morphlineContext.getExceptionHandler().handleException(e, null); } Notifications.notifyShutdown(morphline); } }
@Test public void testNotifications() throws Exception { morphline = createMorphline("test-morphlines/pipeWithTwoBasicCommands"); Notifications.notifyBeginTransaction(morphline); Notifications.notifyStartSession(morphline); Notifications.notifyCommitTransaction(morphline); Notifications.notifyRollbackTransaction(morphline); }
assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION);
@Test public void testTokenizeText() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "tokenizeText"); for (int i = 0; i < 3; i++) { Record record = new Record(); record.put(Fields.MESSAGE, "Hello World!"); record.put(Fields.MESSAGE, "\nFoo@Bar.com #%()123"); Record expected = record.copy(); expected.getFields().putAll("tokens", Arrays.asList("hello", "world", "foo", "bar.com", "123")); collector.reset(); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); assertEquals(expected, collector.getFirstRecord()); } }
@Test public void testLoadSolrBasic() throws Exception { //System.setProperty("ENV_SOLR_HOME", testSolrHome + File.separator + "collection1"); morphline = createMorphline("test-morphlines" + File.separator + "loadSolrBasic"); //System.clearProperty("ENV_SOLR_HOME"); Record record = new Record(); record.put(Fields.ID, "id0"); record.put("first_name", "Nadja"); // will be sanitized startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); Record expected = new Record(); expected.put(Fields.ID, "id0"); assertEquals(Arrays.asList(expected), collector.getRecords()); assertEquals(1, queryResultSetSize("*:*")); Notifications.notifyRollbackTransaction(morphline); Notifications.notifyShutdown(morphline); }
@Test public void testLoadSolrWithChildDocuments() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "loadSolrWithChildDocuments"); Record record = new Record(); record.put(Fields.ID, "id0"); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); // This parent block join returns the parent records for records // where the child documents contain "bar" in the id field. SolrDocumentList docs = query("{!parent which='content_type:parent'}id:bar").getResults(); assertEquals(1, docs.size()); assertEquals("id0", docs.get(0).getFirstValue(Fields.ID)); docs = query("*:*").getResults(); assertEquals(3, docs.size()); }
Notifications.notifyCommitTransaction(morphline); new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION);
assertEquals(Arrays.asList("hello sun", "goodbye mars"), docs.get(0).get("text")); Notifications.notifyCommitTransaction(morphline); Notifications.notifyShutdown(morphline);
expected2.put("user_screen_name", "foo1"); Notifications.notifyCommitTransaction(morphline); new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION);