Refine search
@Override public Schema onSchemaProcessed(Schema rootSchema, Set<Schema> addressableRecords) { List<Schema> deltaTypes = new ArrayList<Schema>(addressableRecords.size() + 1); deltaTypes.add(rootSchema); deltaTypes.addAll(addressableRecords); Field deltaTypesField = new Field(DELTA, Schema.createUnion(deltaTypes), null, null); List<Field> deltaFields = new ArrayList<Field>(); deltaFields.add(deltaTypesField); Schema delta = Schema.createRecord(DELTA + "T", null, KAA_NAMESPACE, false); delta.setFields(deltaFields); return Schema.createArray(delta); }
private static Schema makePairSchema(Schema key, Schema value) { Schema pair = Schema.createRecord(PAIR, null, null, false); List<Field> fields = new ArrayList<>(); fields.add(new Field(KEY, key, "", null)); fields.add(new Field(VALUE, value, "", null, Field.Order.IGNORE)); pair.setFields(fields); return pair; }
/** * Creates a KeyValuePair generic record schema. * * @return A schema for a generic record with two fields: 'key' and * 'value'. */ public static Schema getSchema(Schema keySchema, Schema valueSchema) { Schema schema = Schema.createRecord(KEY_VALUE_PAIR_RECORD_NAME, "A key/value pair", KEY_VALUE_PAIR_RECORD_NAMESPACE, false); schema.setFields(Arrays.asList(new Schema.Field(KEY_FIELD, keySchema, "The key", null), new Schema.Field(VALUE_FIELD, valueSchema, "The value", null))); return schema; } }
private static Schema _getDeleteSchema() { List<Schema.Field> schemaFields = new ArrayList<>(1); Schema.Field designField = new Schema.Field( AvroConstants.ID, AvroUtils._string(), null, (Object)null); designField.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); schemaFields.add(designField); Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields); return schema; }
private List<Schema.Field> getFields(Schema.Field schemaField) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); if (schemaField.schema().getType() == Schema.Type.RECORD) { for (Schema.Field field : schemaField.schema().getFields()) { fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); } } else { fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), nullDefault)); } return fields; }
List<Schema.Field> newFields = new ArrayList<>(fields.size()); for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), copyDocs ? field.doc() : null, field.defaultVal(), field.order()); copyProperties.copy(field, newField); newFields.add(newField); newSchema.setFields(newFields); return SchemaVisitorAction.CONTINUE; case UNION: List<Schema> newTypes = new ArrayList<>(types.size()); for (Schema sch : types) { newTypes.add(replace.get(sch));
List<Schema> newUnionMembers = new ArrayList<>(); if (isNullFirstMember) { newUnionMembers.add(Schema.create(Schema.Type.NULL)); continue; newUnionMembers.add(type); newUnionMembers.add(Schema.create(Schema.Type.NULL)); Schema.Field field = new Schema.Field(flattenName, flattenedFieldSchema, f.doc(), f.defaultValue(), f.order());
@Test public void testIncomingTimeColumn() throws Exception { List<Schema.Field> avroFields = Collections.singletonList(new Schema.Field("incomingTime", Schema.create(Schema.Type.LONG), null, null)); Schema avroSchema = Schema.createRecord(avroFields); GenericData.Record avroRecord = new GenericData.Record(avroSchema); avroRecord.put("incomingTime", 12345L); org.apache.pinot.common.data.Schema pinotSchema = new org.apache.pinot.common.data.Schema.SchemaBuilder().setSchemaName("testSchema") .addTime("incomingTime", TimeUnit.MILLISECONDS, FieldSpec.DataType.LONG, "outgoingTime", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); AvroRecordToPinotRowGenerator avroRecordToPinotRowGenerator = new AvroRecordToPinotRowGenerator(pinotSchema); GenericRow genericRow = new GenericRow(); avroRecordToPinotRowGenerator.transform(avroRecord, genericRow); Assert.assertEquals(genericRow.getFieldNames(), new String[]{"incomingTime"}); Assert.assertEquals(genericRow.getValue("incomingTime"), 12345L); } }
private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Set<Schema> processed) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); if (processed.contains(record)) { return Optional.absent(); } processed.add(record); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false); newSchema.setFields(fields); return Optional.of(newSchema); }
Schema.Field designField = new Schema.Field( fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); schemaFields.add(i, designField); return Schema.createRecord("Runtime", null, null, false, schemaFields);
private static List<Schema.Field> _cloneFieldsAndResetPosition( List<Schema.Field> fields) { List<Schema.Field> copyFieldList = new ArrayList<>(); for (Schema.Field schemaEntry : fields) { Schema.Field field = new Schema.Field( schemaEntry.name(), schemaEntry.schema(), schemaEntry.doc(), schemaEntry.defaultVal(), schemaEntry.order()); Map<String, Object> objectProperties = schemaEntry.getObjectProps(); for (Map.Entry<String, Object> entry : objectProperties.entrySet()) { field.addProp(entry.getKey(), entry.getValue()); } copyFieldList.add(field); } return copyFieldList; }
List<Schema.Field> newFields = new ArrayList<>(fields.size()); for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), field.doc(), field.defaultVal(), field.order()); copyAllProperties(field, newField); newFields.add(newField); newSchema.setFields(newFields); List<Schema> newTypes = new ArrayList<>(types.size()); for (Schema sch : types) { newTypes.add(replace.get(sch));
/** * TODO: Currently generates a static schema avro record. **/ public static GenericRecord generateRandomAvroRecord() { ArrayList<Schema.Field> fields = new ArrayList<Schema.Field>(); String fieldName = "field1"; Schema fieldSchema = Schema.create(Schema.Type.STRING); String docString = "doc"; fields.add(new Schema.Field(fieldName, fieldSchema, docString, null)); Schema schema = Schema.createRecord("name", docString, "test",false); schema.setFields(fields); GenericData.Record record = new GenericData.Record(schema); record.put("field1", "foobar"); return record; }
public void testRemoveUuid() { Schema uuidSchema = Schema.createFixed("uuidT", "", "org.kaaproject.configuration", 16); Schema.Field uuidField = new Schema.Field("__uuid", uuidSchema, "", null); Schema recordSchema = Schema.createRecord(Arrays.asList(uuidField)); Schema.Field otherFixed = new Schema.Field("some_field", otherFixedSchema, "", null); byte[] otherFixedValue = {1}; Schema.Field innerRecordField = new Schema.Field("inner", recordSchema, "", null); Schema.Field intField = new Schema.Field("int_value", intFieldSchema, "", null); Schema.Field uuidInArrayField = new Schema.Field("__uuid", uuidSchema, "", null); Schema inArrayRecordSchema = Schema.createRecord(Arrays.asList(uuidInArrayField, intField)); Schema.Field arrayField = new Schema.Field("array1", arraySchema, "", null); GenericArray<GenericRecord> array1 = new GenericData.Array<GenericRecord>(2, arraySchema); array1.add(inArrayRecord1); array1.add(inArrayRecord2); Schema.Field uuidOfComplexRecordField = new Schema.Field("__uuid", uuidSchema, "", null); Schema complexRecordSchemaWithUuid = Schema.createRecord(Arrays.asList(innerRecordField, arrayField, uuidOfComplexRecordField, otherFixed));
public static Optional<Schema> getKeySchemaFromRecord(Schema record) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = getKeySchema(field); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } if (!fields.isEmpty()) { Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getName(), false); newSchema.setFields(fields); return Optional.of(newSchema); } else { return Optional.absent(); } }
Schema.Field designField = new Schema.Field( fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); schemaFields.add(i, designField); Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields);
if (!field.schema().getType().equals(Schema.Type.MAP)) { if (fieldsRenameMap.containsKey(curFieldName)) { newFields.add( new Schema.Field(fieldsRenameMap.get(curFieldName), field.schema(), field.doc(), field.defaultValue())); } else { newFields.add(new Schema.Field(curFieldName, field.schema(), field.doc(), field.defaultValue())); mapFieldNames.add(curFieldName); String newFieldName = this.fieldsRenameMap.containsKey(fieldToFlatten) ? this.fieldsRenameMap.get(fieldToFlatten) : fieldToFlatten; newFields.add(new Field(newFieldName, Schema.create(Schema.Type.STRING), "", null));
@Test public void testSchemaWithFields() { List<Field> fields = new ArrayList<>(); fields.add(new Field("field_name1", Schema.create(Type.NULL), null, null)); fields.add(new Field("field_name2", Schema.create(Type.INT), null, null)); Schema schema = createDefaultRecord(); schema.setFields(fields); String schemaString = schema.toString(); assertNotNull(schemaString); assertEquals(2, schema.getFields().size()); }
/** * Creates a KeyValuePair generic record schema. * * @return A schema for a generic record with two fields: 'key' and 'value'. */ public static Schema getSchema(Schema keySchema, Schema valueSchema) { Schema schema = Schema.createRecord( KEY_VALUE_PAIR_RECORD_NAME, "A key/value pair", KEY_VALUE_PAIR_RECORD_NAMESPACE, false); schema.setFields(Arrays.asList( new Schema.Field(KEY_FIELD, keySchema, "The key", null), new Schema.Field(VALUE_FIELD, valueSchema, "The value", null))); return schema; }
@Test public void testValidateSchemaNewField() throws Exception { final List<Schema.Field> readerFields = list( new Schema.Field("oldfield1", INT_SCHEMA, null, null), new Schema.Field("newfield1", INT_SCHEMA, null, null)); final Schema reader = Schema.createRecord(readerFields); SchemaPairCompatibility compatibility = checkReaderWriterCompatibility(reader, WRITER_SCHEMA); // Test new field without default value. assertEquals(SchemaCompatibility.SchemaCompatibilityType.INCOMPATIBLE, compatibility.getType()); assertEquals(SchemaCompatibility.SchemaCompatibilityResult.incompatible( SchemaIncompatibilityType.READER_FIELD_MISSING_DEFAULT_VALUE, reader, WRITER_SCHEMA, "newfield1", asList("", "fields", "1")), compatibility.getResult()); assertEquals( String.format( "Data encoded using writer schema:%n%s%n" + "will or may fail to decode using reader schema:%n%s%n", WRITER_SCHEMA.toString(true), reader.toString(true)), compatibility.getDescription()); assertEquals(reader, compatibility.getReader()); assertEquals(WRITER_SCHEMA, compatibility.getWriter()); }