public MorphlineXMLStreamWriter(Command child, Record template) { this.child = child; this.template = template; this.record = template.copy(); }
@Override protected boolean doProcess(Record inputRecord) { JsonNode datum = (JsonNode) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Record outputRecord = inputRecord.copy(); for (Map.Entry<String, Collection<String>> entry : stepMap.entrySet()) { String fieldName = entry.getKey(); List<String> steps = (List<String>) entry.getValue(); extractPath(datum, fieldName, steps, outputRecord, 0); } // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record inputRecord) { JsonNode datum = (JsonNode) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Record outputRecord = inputRecord.copy(); for (Map.Entry<String, Collection<String>> entry : stepMap.entrySet()) { String fieldName = entry.getKey(); List<String> steps = (List<String>) entry.getValue(); extractPath(datum, fieldName, steps, outputRecord, 0); } // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record record) { Record outputRecord = record.copy(); for (Entry<String, ConfigValue> entry : entries) { outputRecord.put(entry.getValue().render().replace("\"", ""), outputRecord.get(entry.getKey()).get(0)); if (remove) { outputRecord.removeAll(entry.getKey()); } } return super.doProcess(outputRecord); } }
@Override protected boolean doProcess(Record record) { Record outputRecord = record.copy(); for (Entry<String, ConfigValue> entry : entries) { outputRecord.put(entry.getValue().render().replace("\"", ""), outputRecord.get(entry.getKey()).get(0)); if (remove) { outputRecord.removeAll(entry.getKey()); } } return super.doProcess(outputRecord); } }
@Override protected boolean doProcess(Record inputRecord) { // Preconditions.checkState(ReadAvroBuilder.AVRO_MEMORY_MIME_TYPE.equals(inputRecord.getFirstValue(Fields.ATTACHMENT_MIME_TYPE))); GenericContainer datum = (GenericContainer) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Preconditions.checkNotNull(datum.getSchema()); Record outputRecord = inputRecord.copy(); for (Map.Entry<String, Collection<String>> entry : stepMap.entrySet()) { String fieldName = entry.getKey(); List<String> steps = (List<String>) entry.getValue(); extractPath(datum, datum.getSchema(), fieldName, steps, outputRecord, 0); } // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record inputRecord) { Record outputRecord = inputRecord.copy(); AbstractParser.removeAttachments(outputRecord); ByteArrayOutputStream bout = new ByteArrayOutputStream(1024); if (format == Format.container) { writeContainer(inputRecord, bout); } else { writeContainerless(inputRecord, bout); } outputRecord.put(Fields.ATTACHMENT_BODY, bout.toByteArray()); // pass record to next command in chain: return super.doProcess(outputRecord); }
@Override protected boolean doProcess(Record inputRecord) { // Preconditions.checkState(ReadAvroBuilder.AVRO_MEMORY_MIME_TYPE.equals(inputRecord.getFirstValue(Fields.ATTACHMENT_MIME_TYPE))); GenericContainer datum = (GenericContainer) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Preconditions.checkNotNull(datum.getSchema()); Record outputRecord = inputRecord.copy(); extractTree(datum, datum.getSchema(), outputRecord, outputFieldPrefix); // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record inputRecord) { // Preconditions.checkState(ReadAvroBuilder.AVRO_MEMORY_MIME_TYPE.equals(inputRecord.getFirstValue(Fields.ATTACHMENT_MIME_TYPE))); GenericContainer datum = (GenericContainer) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Preconditions.checkNotNull(datum.getSchema()); Record outputRecord = inputRecord.copy(); extractTree(datum, datum.getSchema(), outputRecord, outputFieldPrefix); // pass record to next command in chain: return getChild().process(outputRecord); }
private boolean extract(GenericContainer datum, Record inputRecord) { incrementNumRecords(); Record outputRecord = inputRecord.copy(); outputRecord.put(Fields.ATTACHMENT_BODY, datum); // pass record to next command in chain: return getChild().process(outputRecord); }
protected boolean extract(GenericContainer datum, Record inputRecord) { incrementNumRecords(); Record outputRecord = inputRecord.copy(); outputRecord.put(Fields.ATTACHMENT_BODY, datum); // pass record to next command in chain: return getChild().process(outputRecord); } }
private boolean extract(GenericContainer datum, Record inputRecord) { incrementNumRecords(); Record outputRecord = inputRecord.copy(); outputRecord.put(Fields.ATTACHMENT_BODY, datum); // pass record to next command in chain: return getChild().process(outputRecord); }
protected boolean extract(GenericContainer datum, Record inputRecord) { incrementNumRecords(); Record outputRecord = inputRecord.copy(); outputRecord.put(Fields.ATTACHMENT_BODY, datum); // pass record to next command in chain: return getChild().process(outputRecord); } }
@Override protected boolean doProcess(Record record, InputStream stream) throws IOException { Record outputRecord = record.copy(); BigDecimal result = doComputation(outputRecord); outputRecord.put(outField, result); return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record record, InputStream stream) throws IOException { Record outputRecord = record.copy(); BigDecimal result = doComputation(outputRecord); outputRecord.put(outField, result); return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record record) { for (int i = 0; i < count; i++) { Record next = record.copy(); next.replaceValues(name, i); if (!getChild().process(next)) { return false; } } return true; }
private void processAndVerifyUnion(Object input, Object expected, List<Schema> types) { Schema documentSchema = Schema.createRecord("Doc", "adoc", null, false); Schema unionSchema = Schema.createUnion(types); documentSchema.setFields(Arrays.asList(new Field("price", unionSchema, null, null))); GenericData.Record document1 = new GenericData.Record(documentSchema); document1.put("price", expected); Record jdoc1 = new Record(); jdoc1.put("_dataset_descriptor_schema", documentSchema); jdoc1.put("price", input); Record expect1 = jdoc1.copy(); expect1.put(Fields.ATTACHMENT_BODY, document1); processAndVerifySuccess(jdoc1, expect1, false); }
@Test public void testConvertHTMLBlogThenRunXSQLT() throws Exception { morphline = createMorphline("test-morphlines/convertHTMLBlogThenRunXSLT"); byte[] bytes = Files.toByteArray(new File(RESOURCES_DIR + "/test-documents/blog.html")); Record record = new Record(); record.put("id", "123"); record.put(Fields.ATTACHMENT_BODY, bytes); for (int i = 0; i < 3; i++) { assertTrue(morphline.process(record.copy())); // TODO check details } }
private void testAddLocalHostInternal(String name) throws Exception { Record record = new Record(); Record expected = new Record(); expected.put("myhost", name); processAndVerifySuccess(record, expected); // test that preserveExisting = true preserves the existing value record = new Record(); record.put("myhost", "myname"); expected = record.copy(); processAndVerifySuccess(record, expected); }
@Override protected boolean doProcess(Record inputRecord) { Record outputRecord = inputRecord.copy(); // change key Text myTextKey = (Text)inputRecord.getFirstValue(this.keyField); outputRecord.replaceValues(this.keyField, MyWritable.keyStr(myTextKey)); // change value MyWritable myWritableValue = (MyWritable)inputRecord.getFirstValue(this.valueField); outputRecord.replaceValues(this.valueField, MyWritable.valueStr(myWritableValue)); return super.doProcess(outputRecord); } }