Refine search
@Override public GenericRecord partitionForRecord(GenericRecord record) { GenericRecord partition = new GenericData.Record(SCHEMA); partition.put(TITLE, record.get("title")); return partition; } }
private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { final List<Schema.Field> fields = schema.getFields(); final int length = fields.size(); final GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < length; i++) { final Schema.Field field = fields.get(i); record.put(i, convertFlinkType(field.schema(), row.getField(i))); } return record; }
/** * @param message kafka message in the string format * @return GenericRecord that is deserialized from kafka message w.r.t. expected schema */ public static GenericRecord genericRecordFromJson(String message) { GenericRecord record = new GenericData.Record(DefaultTopicSchema.MESSAGE_V0); JSONObject jsonObject = new JSONObject(message); record.put(DefaultTopicSchema.TOPIC_FIELD.name(), jsonObject.getString(DefaultTopicSchema.TOPIC_FIELD.name())); record.put(DefaultTopicSchema.INDEX_FIELD.name(), jsonObject.getLong(DefaultTopicSchema.INDEX_FIELD.name())); record.put(DefaultTopicSchema.TIME_FIELD.name(), jsonObject.getLong(DefaultTopicSchema.TIME_FIELD.name())); record.put(DefaultTopicSchema.PRODUCER_ID_FIELD.name(), jsonObject.getString(DefaultTopicSchema.PRODUCER_ID_FIELD.name())); record.put(DefaultTopicSchema.CONTENT_FIELD.name(), jsonObject.getString(DefaultTopicSchema.CONTENT_FIELD.name())); return record; }
@Override public Iterable<GenericRecord> convertRecord(Schema schema, String inputRecord, WorkUnitState workUnit) throws DataConversionException { JsonElement element = GSON.fromJson(inputRecord, JsonElement.class); Map<String, Object> fields = GSON.fromJson(element, FIELD_ENTRY_TYPE); GenericRecord record = new GenericData.Record(schema); for (Map.Entry<String, Object> entry : fields.entrySet()) { record.put(entry.getKey(), entry.getValue()); } return new SingleRecordIterable<>(record); } }
@Override public GenericRecord partitionForRecord(GenericRecord record) { GenericRecord partition = new GenericData.Record(SCHEMA); partition.put(SCHEMA_STRING, record.getSchema().toString()); return partition; } }
@Override public GenericRecord partitionForRecord(ParquetGroup record) { GenericRecord partition = new GenericData.Record(SCHEMA); partition.put(PARTITION_KEY, record.getString(PARTITION_KEY, 0).replace("\"", "")); return partition; } }
@Override public ByteBuffer write(List<Object> data, ByteBuffer buffer) { Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schemas"); try { Schema schema = schemas.getSchema(schemaString); GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < fieldNames.size(); i++) { record.put(fieldNames.get(i), data.get(i)); } ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema()); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(record, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); out.close(); return ByteBuffer.wrap(bytes); } catch (IOException e) { throw new RuntimeException(e); } } }
@Override public Iterable<GenericRecord> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { GenericRecord outputRecord = new GenericData.Record(outputSchema); for (Field field : inputRecord.getSchema().getFields()) { outputRecord.put(field.name(), convertFieldValue(outputSchema, field, inputRecord, workUnit)); } return new SingleRecordIterable<>(outputRecord); }
@Override void genSourceData() { Random r = newRandom(); sourceData = new GenericRecord[count]; for (int i = 0; i < sourceData.length; i++) { GenericRecord rec = new GenericData.Record(schema); rec.put(0, randomString(r)); rec.put(1, randomString(r)); rec.put(2, randomString(r)); sourceData[i] = rec; } } }
@Override public GenericRecord partitionForRecord(String record) { GenericRecord partition = new GenericData.Record(SCHEMA); partition.put(PARTITION, record.toLowerCase().charAt(0)); return partition; } }
private GenericRecord aDatum(Type ofType, int forRow) { GenericRecord record; switch (ofType) { case STRING: record = new GenericData.Record(STRINGSCHEMA); record.put("value", String.valueOf(forRow % 100)); return record; case INT: record = new GenericData.Record(INTSCHEMA); record.put("value", forRow); return record; default: throw new AssertionError("I can't generate data for this type"); } }
private GenericRecord convertRecord(String inputRecord) { Gson gson = new Gson(); JsonElement element = gson.fromJson(inputRecord, JsonElement.class); Map<String, Object> fields = gson.fromJson(element, FIELD_ENTRY_TYPE); GenericRecord outputRecord = new GenericData.Record(this.schema); for (Map.Entry<String, Object> entry : fields.entrySet()) { outputRecord.put(entry.getKey(), entry.getValue()); } return outputRecord; } }
@Test public void testCopy() throws CopyNotSupportedException { GenericRecord record = new GenericData.Record(new Schema.Parser().parse(AVRO_SCHEMA)); record.put("name", "foo"); record.put("favorite_number", 68); record.put("favorite_colors", Arrays.asList("blue", "black", "red")); CopyableGenericRecord copyableGenericRecord = new CopyableGenericRecord(record); GenericRecord copy = copyableGenericRecord.copy(); Assert.assertEquals(record, copy); copy.put("name", "bar"); Assert.assertNotEquals(record, copy); } }
@Override public Iterable<GenericRecord> convertRecord(Schema schema, String inputRecord, WorkUnitState workUnit) { JsonElement element = GSON.fromJson(inputRecord, JsonElement.class); Map<String, Object> fields = GSON.fromJson(element, FIELD_ENTRY_TYPE); GenericRecord record = new GenericData.Record(schema); for (Map.Entry<String, Object> entry : fields.entrySet()) { record.put(entry.getKey(), entry.getValue()); } return new SingleRecordIterable<GenericRecord>(record); } }
private void output(final AvroOutputFormat<GenericRecord> outputFormat, Schema schema) throws IOException { outputFormat.configure(new Configuration()); outputFormat.open(1, 1); for (int i = 0; i < 100; i++) { GenericRecord record = new GenericData.Record(schema); record.put("user_name", "testUser"); record.put("favorite_number", 1); record.put("favorite_color", "blue"); outputFormat.writeRecord(record); } outputFormat.close(); } }
@Test public void testSerializeAsPath() throws Exception { Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"test\", " + "\"fields\":[" + "{\"name\": \"name\", \"type\": \"string\"}, " + "{\"name\": \"title\", \"type\": \"string\"}" + "]}"); GenericRecord partition = new GenericData.Record(schema); partition.put("name", "a/b:c\\d e"); partition.put("title", "title"); Assert.assertEquals(AvroUtils.serializeAsPath(partition, true, true), new Path("name=a_b_c_d_e/title=title")); Assert.assertEquals(AvroUtils.serializeAsPath(partition, false, true), new Path("a_b_c_d_e/title")); Assert.assertEquals(AvroUtils.serializeAsPath(partition, false, false), new Path("a/b_c_d_e/title")); }
private GenericRow serializeDeserializeRow(final Schema schema, final String topicName, final SchemaRegistryClient schemaRegistryClient, final org.apache.avro.Schema rowAvroSchema, final GenericRow genericRow) { final GenericRecord avroRecord = new GenericData.Record(rowAvroSchema); final List<org.apache.avro.Schema.Field> fields = rowAvroSchema.getFields(); for (int i = 0; i < genericRow.getColumns().size(); i++) { avroRecord.put(fields.get(i).name(), genericRow.getColumns().get(i)); } return serializeDeserializeAvroRecord(schema, topicName, schemaRegistryClient, avroRecord); }
public static void fillComplexFullResyncDelta(GenericRecord delta) { GenericRecord testField2 = new GenericData.Record(getSchemaByFullName( delta.getSchema().getField("testField2").schema().getTypes(), "org.kaa.config.testRecordT")); testField2.put("testField3", 456); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta.getSchema() .getField("__uuid").schema(), rawUuid); delta.put("testField1", "abc"); delta.put("testField2", testField2); delta.put("__uuid", uuid); }
@Override void genSourceData() { Random r = newRandom(); Schema uSchema = writeSchema.getField("f").schema(); sourceData = new GenericRecord[count]; for (int i = 0; i < sourceData.length; i++) { GenericRecord rec = new GenericData.Record(writeSchema); int val = r.nextInt(1000000); Integer v = (val < 750000 ? new Integer(val) : null); rec.put("f", v); sourceData[i] = rec; } } }
@Test public void testConvertGenericToSpecific() { GenericRecord generic = new GenericData.Record(TestRecord.SCHEMA$); generic.put("name", "foo"); generic.put("kind", new GenericData.EnumSymbol(Kind.SCHEMA$, "BAR")); generic.put("hash", new GenericData.Fixed (MD5.SCHEMA$, new byte[]{0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5})); TestRecord specific = (TestRecord)SpecificData.get().deepCopy(TestRecord.SCHEMA$, generic); }