@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
@Override public boolean process(Record record) { Preconditions.checkNotNull(record); return child.process(record); }
@Override protected boolean doProcess(Record record) { if (LOG.isDebugEnabled()) { return super.doProcess(record); } else { return getChild().process(record); } }
@Override protected boolean doProcess(Record record) { if (LOG.isInfoEnabled()) { return super.doProcess(record); } else { return getChild().process(record); } }
@Override protected boolean doProcess(Record record) { if (LOG.isTraceEnabled()) { return super.doProcess(record); } else { return getChild().process(record); } }
@Override protected boolean doProcess(Record record) { if (LOG.isErrorEnabled()) { return super.doProcess(record); } else { return getChild().process(record); } }
@Override protected boolean doProcess(Record inputRecord) { // Preconditions.checkState(ReadAvroBuilder.AVRO_MEMORY_MIME_TYPE.equals(inputRecord.getFirstValue(Fields.ATTACHMENT_MIME_TYPE))); GenericContainer datum = (GenericContainer) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Preconditions.checkNotNull(datum.getSchema()); Record outputRecord = inputRecord.copy(); extractTree(datum, datum.getSchema(), outputRecord, outputFieldPrefix); // pass record to next command in chain: return getChild().process(outputRecord); }
protected boolean extract(GenericContainer datum, Record inputRecord) { incrementNumRecords(); Record outputRecord = inputRecord.copy(); outputRecord.put(Fields.ATTACHMENT_BODY, datum); // pass record to next command in chain: return getChild().process(outputRecord); } }
@Override protected boolean doProcess(Record record) { for (int i = 0; i < count; i++) { Record next = record.copy(); next.replaceValues(name, i); if (!getChild().process(next)) { return false; } } return true; }
@Test public void testSampleWithoutSeed() throws Exception { morphline = createMorphline("test-morphlines/sampleWithoutSeed"); int size = 100; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } int actual = collector.getRecords().size(); double delta = 0.2; assertTrue("actual: " + actual, actual >= Math.round(size * (0.9 - delta))); assertTrue("actual: " + actual, actual <= Math.round(size * (0.9 + delta))); }
@Test public void testHeadUnlimited() throws Exception { morphline = createMorphline("test-morphlines/headUnlimited"); int size = 5; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(size, collector.getRecords().size()); }
@Test public void testHead() throws Exception { morphline = createMorphline("test-morphlines/head"); int size = 5; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(3, collector.getRecords().size()); }
private void processAndVerifyFailure(Record input) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(input)); assertEquals(0, collector.getRecords().size()); }
@Test public void testGrokWithEscaping() throws Exception { morphline = createMorphline("test-morphlines/grokWithEscaping"); Record record = new Record(); record.put(Fields.MESSAGE, "{SystemSourceId}=foo"); assertTrue(morphline.process(record)); assertSame(record, collector.getFirstRecord()); record = new Record(); record.put(Fields.MESSAGE, "SystemSourceId}=foo"); // missing opening brace assertFalse(morphline.process(record)); }
@Test public void testNotWithTrue() throws Exception { morphline = createMorphline("test-morphlines/notWithTrue"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(record)); assertEquals(Lists.newArrayList(), collector.getRecords()); }
private void testGenerateUUID(String suffix) throws Exception { morphline = createMorphline("test-morphlines/generateUUID" + suffix); Record record = new Record(); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record actual = collector.getFirstRecord(); assertEquals(1, actual.get("id").size()); String uuid = (String) actual.getFirstValue("id"); assertEquals(36, uuid.length()); }
@Test public void testDetectMimeTypesWithDefaultMimeTypes() throws Exception { morphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypes"); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); startSession(); morphline.process(record); assertEquals("image/jpeg", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }
@Test public void testDecodeBase64() throws Exception { morphline = createMorphline("test-morphlines/decodeBase64"); Record record = new Record(); record.put("data", "SGVsbG8gV29ybGQ="); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); byte[] actual = (byte[]) collector.getFirstRecord().getFirstValue("data"); assertArrayEquals("Hello World".getBytes(Charsets.UTF_8), actual); assertSame(record, collector.getFirstRecord()); }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }