@Override public StructuredRecord make(Schema schema, Iterator<String> bodyFields) { StructuredRecord.Builder builder = StructuredRecord.builder(schema); List<String> fields = Lists.newArrayList(bodyFields); builder.set("body", fields.toArray(new String[fields.size()])); return builder.build(); } }
@Override public StructuredRecord make(Schema schema, Iterator<String> bodyFields) { StructuredRecord.Builder builder = StructuredRecord.builder(schema); List<String> fields = Lists.newArrayList(bodyFields); builder.set("body", fields.toArray(new String[fields.size()])); return builder.build(); } }
public StructuredRecord.Builder transform(GenericRecord genericRecord, Schema structuredSchema, @Nullable String skipField) throws IOException { StructuredRecord.Builder builder = StructuredRecord.builder(structuredSchema); for (Schema.Field field : structuredSchema.getFields()) { String fieldName = field.getName(); if (!fieldName.equals(skipField)) { builder.set(fieldName, convertField(genericRecord.get(fieldName), field.getSchema())); } } return builder; }
@Override public StructuredRecord read(ByteBuffer input) { String bodyAsStr = Bytes.toString(input, charset); return StructuredRecord.builder(schema).set(fieldName, bodyAsStr).build(); }
public static StructuredRecord convertToRecord(MapWritable input, Schema schema) throws IOException { StructuredRecord.Builder builder = StructuredRecord.builder(schema); for (Schema.Field field : schema.getFields()) { try { builder.set(field.getName(), convertWritables(input.get(new Text(field.getName())), field.getSchema())); } catch (Exception e) { throw(new IOException(String.format("Type exception for field %s: %s", field.getName(), e.getMessage()))); } } return builder.build(); }
@Override public StructuredRecord read(StreamEvent event) { String bodyAsStr = Bytes.toString(event.getBody(), charset); return StructuredRecord.builder(schema).set(fieldName, bodyAsStr).build(); }
@Override public StructuredRecord transform(GenericRecord genericRecord, Schema structuredSchema) throws IOException { StructuredRecord.Builder builder = StructuredRecord.builder(structuredSchema); for (Schema.Field field : structuredSchema.getFields()) { String fieldName = field.getName(); builder.set(fieldName, convertField(genericRecord.get(fieldName), field.getSchema())); } return builder.build(); }
private StructuredRecord decodeRecord(Map nativeObject, Schema schema) { StructuredRecord.Builder builder = StructuredRecord.builder(schema); for (Schema.Field field : schema.getFields()) { String fieldName = field.getName(); Object fieldVal = nativeObject.get(fieldName); builder.set(fieldName, decode(fieldVal, field.getSchema())); } return builder.build(); }
private StructuredRecord decodeRecord(Map nativeObject, Schema schema) { StructuredRecord.Builder builder = StructuredRecord.builder(schema); for (Schema.Field field : schema.getFields()) { String fieldName = field.getName(); Object fieldVal = nativeObject.get(fieldName); builder.set(fieldName, decode(fieldVal, field.getSchema())); } return builder.build(); }
private StructuredRecord decodeRecord(String name, JsonObject nativeObject, Schema schema) throws RecordConvertorException { StructuredRecord.Builder builder = StructuredRecord.builder(schema); for (Schema.Field field : schema.getFields()) { String fieldName = field.getName(); Object fieldVal = nativeObject.get(fieldName); builder.set(fieldName, decode(name, fieldVal, field.getSchema())); } return builder.build(); }
private StructuredRecord decodeRecord(String name, Map nativeObject, Schema schema) throws RecordConvertorException { StructuredRecord.Builder builder = StructuredRecord.builder(schema); for (Schema.Field field : schema.getFields()) { String fieldName = field.getName(); Object fieldVal = nativeObject.get(fieldName); builder.set(fieldName, decode(name, fieldVal, field.getSchema())); } return builder.build(); }
private StructuredRecord.Builder getOutputBuilder(StructuredRecord input) { List<Schema.Field> outFields = new ArrayList<>(); for (Schema.Field field : input.getSchema().getFields()) { outFields.add(field); } Schema outSchema = Schema.recordOf(input.getSchema().getRecordName(), outFields); // copy all the values StructuredRecord.Builder outputBuilder = StructuredRecord.builder(outSchema); for (Schema.Field inField : input.getSchema().getFields()) { outFields.add(inField); outputBuilder.set(inField.getName(), input.get(inField.getName())); } return outputBuilder; }
@Override public void transform(KeyValue<byte[], byte[]> input, Emitter<StructuredRecord> emitter) throws Exception { emitter.emit(StructuredRecord.builder(SCHEMA).set("key", input.getKey()).set("value", input.getValue()).build()); } }
@Override public void groupBy(StructuredRecord record, Emitter<StructuredRecord> emitter) throws Exception { if (uniqueFields == null) { emitter.emit(record); return; } StructuredRecord.Builder builder = StructuredRecord.builder(getGroupKeySchema(record.getSchema())); for (String fieldName : uniqueFields) { builder.set(fieldName, record.get(fieldName)); } emitter.emit(builder.build()); }
@Override public void groupBy(StructuredRecord record, Emitter<StructuredRecord> emitter) throws Exception { // app should provide some way to make some data calculated in configurePipeline available here. // then we wouldn't have to calculate schema here StructuredRecord.Builder builder = StructuredRecord.builder(getGroupKeySchema(record.getSchema())); for (String groupByField : conf.getGroupByFields()) { builder.set(groupByField, record.get(groupByField)); } emitter.emit(builder.build()); }
@Override public void groupBy(StructuredRecord record, Emitter<StructuredRecord> emitter) { if (fields == null) { emitter.emit(record); return; } Schema recordSchema = outputSchema == null ? getOutputSchema(record.getSchema(), fields) : outputSchema; StructuredRecord.Builder builder = StructuredRecord.builder(recordSchema); for (String fieldName : fields) { builder.set(fieldName, record.get(fieldName)); } emitter.emit(builder.build()); }
@Override public StructuredRecord merge(StructuredRecord joinKey, Iterable<JoinElement<StructuredRecord>> joinRow) { StructuredRecord.Builder outRecordBuilder; outRecordBuilder = StructuredRecord.builder(outputSchema); for (JoinElement<StructuredRecord> joinElement : joinRow) { StructuredRecord record = joinElement.getInputRecord(); for (Schema.Field field : record.getSchema().getFields()) { outRecordBuilder.set(field.getName(), record.get(field.getName())); } } return outRecordBuilder.build(); }
@Override public void transform(StructuredRecord input, Emitter<StructuredRecord> emitter) throws Exception { Schema outSchema = config.getOutputSchema(input.getSchema()); StructuredRecord.Builder outputBuilder = StructuredRecord.builder(outSchema); for (Schema.Field inField : input.getSchema().getFields()) { outputBuilder.set(config.prefix + inField.getName(), input.get(inField.getName())); } emitter.emit(outputBuilder.build()); }
public static StructuredRecord transform(StructuredRecord record, Schema schema) throws IOException { StructuredRecord.Builder builder = StructuredRecord.builder(schema); for (Schema.Field field : record.getSchema().getFields()) { String name = field.getName(); // If the field name is not in the output, then skip it and if it's not nullable, then // it would be error out -- in which case, the user has to fix the schema to proceed. if (schema.getField(name) != null) { builder.set(name, convertField(record.get(name), field.getSchema())); } } return builder.build(); }
@Override public void transform(ErrorRecord<StructuredRecord> input, Emitter<StructuredRecord> emitter) throws Exception { StructuredRecord invalidRecord = input.getRecord(); StructuredRecord.Builder output = StructuredRecord.builder(getOutputSchema(invalidRecord.getSchema())); for (Schema.Field field : invalidRecord.getSchema().getFields()) { output.set(field.getName(), invalidRecord.get(field.getName())); } emitter.emit(output.set("errMsg", input.getErrorMessage()) .set("errCode", input.getErrorCode()) .set("errStage", input.getStageName()) .build()); }