private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private String detect(Record event, boolean includeMetaData, boolean excludeParameters) throws IOException { List key = Arrays.asList(includeMetaData, excludeParameters); Command cachedMorphline = morphlineCache.get(key); if (cachedMorphline == null) { // avoid recompiling time and again (performance) Config override = ConfigFactory.parseString("INCLUDE_META_DATA : " + includeMetaData + "\nEXCLUDE_PARAMETERS : " + excludeParameters); cachedMorphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypesAndFile", override); morphlineCache.put(key, cachedMorphline); } collector.reset(); assertTrue(cachedMorphline.process(event)); String mimeType = (String) collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE); return mimeType; }
@Test public void testNotWithFalseWithContinuation() throws Exception { morphline = createMorphline("test-morphlines/notWithFalseAndContinuation"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(record, collector.getFirstRecord()); assertSame(record, collector.getFirstRecord()); assertEquals("touched", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testExtractAvroPathsArrayInUnion() throws Exception { List<String> items = Arrays.asList("a", "b", "c"); ArrayInUnionTestRecord avroRecord = new ArrayInUnionTestRecord(items, items); morphline = createMorphline("test-morphlines/extractAvroPathsArrayInUnion"); deleteAllDocuments(); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, avroRecord); startSession(); assertTrue(morphline.process(record)); assertEquals(1, collector.getRecords().size()); assertEquals(Arrays.asList(items), collector.getFirstRecord().get("/items[]")); assertEquals(Arrays.asList(items), collector.getFirstRecord().get("/itemsInUnion[]")); }
@Test public void testDecodeBase64() throws Exception { morphline = createMorphline("test-morphlines/decodeBase64"); Record record = new Record(); record.put("data", "SGVsbG8gV29ybGQ="); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); byte[] actual = (byte[]) collector.getFirstRecord().getFirstValue("data"); assertArrayEquals("Hello World".getBytes(Charsets.UTF_8), actual); assertSame(record, collector.getFirstRecord()); }
@Test public void testToByteArray() throws Exception { morphline = createMorphline("test-morphlines/toByteArray"); Record record = new Record(); record.put("first_name", "Nadja"); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(1, collector.getFirstRecord().getFields().size()); byte[] expected = "Nadja".getBytes("UTF-8"); assertArrayEquals(expected, (byte[]) collector.getFirstRecord().getFirstValue("first_name")); assertSame(record, collector.getFirstRecord()); }
@Test public void testNotWithFalse() throws Exception { morphline = createMorphline("test-morphlines/notWithFalse"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(record, collector.getFirstRecord()); assertSame(record, collector.getFirstRecord()); }
@Test public void testIfThenElseWithThenEmpty() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThenEmpty"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("init1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithElseEmpty() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithElseEmpty"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("init1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithElse() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithElse"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("else1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithThenAndNot() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThenAndNot"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("then1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testGrokWithMultipleGroupsWithSameName() throws Exception { morphline = createMorphline("test-morphlines/grokWithMultipleGroupsWithSameName"); Record record = new Record(); record.put(Fields.MESSAGE, "123 456"); assertTrue(morphline.process(record)); Record expected = new Record(); expected.put(Fields.MESSAGE, "123 456"); expected.put("syslog_pri", "123"); expected.put("syslog_pri", "456"); assertEquals(expected, collector.getFirstRecord()); assertNotSame(record, collector.getFirstRecord()); }
@Test public void testIfThenElseWithThen() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThen"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("then1", collector.getFirstRecord().getFirstValue("state")); }
private void testGenerateUUID(String suffix) throws Exception { morphline = createMorphline("test-morphlines/generateUUID" + suffix); Record record = new Record(); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record actual = collector.getFirstRecord(); assertEquals(1, actual.get("id").size()); String uuid = (String) actual.getFirstValue("id"); assertEquals(36, uuid.length()); }
@Test public void testDetectMimeTypesWithDefaultMimeTypes() throws Exception { morphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypes"); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); startSession(); morphline.process(record); assertEquals("image/jpeg", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }
private void testDetectMimeTypesInternal(String configFile) throws Exception { // verify that Avro is classified as Avro morphline = createMorphline(configFile); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(AVRO_FILE)); startSession(); morphline.process(record); assertEquals(AVRO_MIME_TYPE, collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); // verify that JPG isnt' classified as JPG because this morphline uses includeDefaultMimeTypes : false collector.reset(); record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); startSession(); morphline.process(record); assertEquals("application/octet-stream", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }
@Test public void testMimeTypeAlreadySpecifiedOnInputRemainsUnchanged() throws Exception { morphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypes"); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); record.put(Fields.ATTACHMENT_MIME_TYPE, "foo/bar"); startSession(); morphline.process(record); assertEquals("foo/bar", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }
@Test public void testGrokWithEscaping() throws Exception { morphline = createMorphline("test-morphlines/grokWithEscaping"); Record record = new Record(); record.put(Fields.MESSAGE, "{SystemSourceId}=foo"); assertTrue(morphline.process(record)); assertSame(record, collector.getFirstRecord()); record = new Record(); record.put(Fields.MESSAGE, "SystemSourceId}=foo"); // missing opening brace assertFalse(morphline.process(record)); }