@Override public void beginTransaction() { Notifications.notifyBeginTransaction(morphline); }
@Override public void beginTransaction() { Notifications.notifyBeginTransaction(morphline); }
Notifications.notifyBeginTransaction(morphline);
Notifications.notifyBeginTransaction(morphline);
MetricRegistry.name("morphline.app", "numExceptionRecords")); Notifications.notifyBeginTransaction(morphline);
MetricRegistry.name("morphline.app", "numExceptionRecords")); Notifications.notifyBeginTransaction(morphline);
MetricRegistry.name(Metrics.MORPHLINE_APP, Metrics.NUM_EXCEPTION_RECORDS)); Notifications.notifyBeginTransaction(morphline);
Notifications.notifyBeginTransaction(pipeline.getMorphline());
/** Usage: java ... <morphline.conf> <dataFile1> ... <dataFileN> */ public static void main(String[] args) throws IOException { // compile morphline.conf file on the fly File morphlineFile = new File(args[0]); String morphlineId = null; MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null); // process each input data file Notifications.notifyBeginTransaction(morphline); try { for (int i = 1; i < args.length; i++) { InputStream in = new BufferedInputStream(new FileInputStream(new File(args[i]))); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); Notifications.notifyStartSession(morphline); boolean success = morphline.process(record); if (!success) { System.out.println("Morphline failed to process record: " + record); } in.close(); } Notifications.notifyCommitTransaction(morphline); } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); morphlineContext.getExceptionHandler().handleException(e, null); } Notifications.notifyShutdown(morphline); } }
@Test public void testNotifications() throws Exception { morphline = createMorphline("test-morphlines/pipeWithTwoBasicCommands"); Notifications.notifyBeginTransaction(morphline); Notifications.notifyStartSession(morphline); Notifications.notifyCommitTransaction(morphline); Notifications.notifyRollbackTransaction(morphline); }
collector.reset(); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents());
private void runTweetContainer(String morphlineConfigFile, String[] fieldNames) throws Exception { File file = new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433-medium.avro"); morphline = createMorphline(morphlineConfigFile); for (int j = 0; j < 3; j++) { // also test reuse of objects and low level avro buffers Record record = new Record(); byte[] body = Files.toByteArray(file); record.put(Fields.ATTACHMENT_BODY, body); collector.reset(); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); assertEquals(2104, collector.getRecords().size()); FileReader<GenericData.Record> reader = new DataFileReader(file, new GenericDatumReader()); int i = 0; while (reader.hasNext()) { Record actual = collector.getRecords().get(i); GenericData.Record expected = reader.next(); assertTweetEquals(expected, actual, fieldNames, i); i++; } assertEquals(collector.getRecords().size(), i); } }
record.put(Fields.ATTACHMENT_BODY, body); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents());
@Test public void testTokenizeText() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "tokenizeText"); for (int i = 0; i < 3; i++) { Record record = new Record(); record.put(Fields.MESSAGE, "Hello World!"); record.put(Fields.MESSAGE, "\nFoo@Bar.com #%()123"); Record expected = record.copy(); expected.getFields().putAll("tokens", Arrays.asList("hello", "world", "foo", "bar.com", "123")); collector.reset(); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); assertEquals(expected, collector.getFirstRecord()); } }
@Test public void testLoadSolrBasic() throws Exception { //System.setProperty("ENV_SOLR_HOME", testSolrHome + File.separator + "collection1"); morphline = createMorphline("test-morphlines" + File.separator + "loadSolrBasic"); //System.clearProperty("ENV_SOLR_HOME"); Record record = new Record(); record.put(Fields.ID, "id0"); record.put("first_name", "Nadja"); // will be sanitized startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); Record expected = new Record(); expected.put(Fields.ID, "id0"); assertEquals(Arrays.asList(expected), collector.getRecords()); assertEquals(1, queryResultSetSize("*:*")); Notifications.notifyRollbackTransaction(morphline); Notifications.notifyShutdown(morphline); }
@Test public void testLoadSolrWithChildDocuments() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "loadSolrWithChildDocuments"); Record record = new Record(); record.put(Fields.ID, "id0"); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); // This parent block join returns the parent records for records // where the child documents contain "bar" in the id field. SolrDocumentList docs = query("{!parent which='content_type:parent'}id:bar").getResults(); assertEquals(1, docs.size()); assertEquals("id0", docs.get(0).getFirstValue(Fields.ID)); docs = query("*:*").getResults(); assertEquals(3, docs.size()); }
startSession(); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record));
record.put("user_friends_count", 123); record.put("text", "myText"); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getRecords().size());
startSession(); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record));
record.replaceValues(Fields.ID, "id0"); record.replaceValues("first_name", "Nadja"); // will be sanitized Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record.copy())); assertEquals(1, query("*:*").getResults().size());