Refine search
/** Create an anonymous record schema. */ public static Schema createRecord(List<Field> fields) { Schema result = createRecord(null, null, null, false); result.setFields(fields); return result; }
List<Schema.Field> newFields = new ArrayList<>(fields.size()); for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), copyDocs ? field.doc() : null, field.defaultVal(), field.order()); copyProperties.copy(field, newField); newFields.add(newField); newSchema.setFields(newFields); return SchemaVisitorAction.CONTINUE; case UNION:
/** Create an anonymous record schema. */ public static Schema createRecord(List<Field> fields) { Schema result = createRecord(null, null, null, false); result.setFields(fields); return result; }
List<Schema.Field> newFields = new ArrayList<>(fields.size()); for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), field.doc(), field.defaultVal(), field.order()); copyAllProperties(field, newField); newFields.add(newField); newSchema.setFields(newFields);
/** * Creates a KeyValuePair generic record schema. * * @return A schema for a generic record with two fields: 'key' and * 'value'. */ public static Schema getSchema(Schema keySchema, Schema valueSchema) { Schema schema = Schema.createRecord(KEY_VALUE_PAIR_RECORD_NAME, "A key/value pair", KEY_VALUE_PAIR_RECORD_NAMESPACE, false); schema.setFields(Arrays.asList(new Schema.Field(KEY_FIELD, keySchema, "The key", null), new Schema.Field(VALUE_FIELD, valueSchema, "The value", null))); return schema; } }
@Test public void testSchemaWithFields() { List<Field> fields = new ArrayList<>(); fields.add(new Field("field_name1", Schema.create(Type.NULL), null, null)); fields.add(new Field("field_name2", Schema.create(Type.INT), null, null)); Schema schema = createDefaultRecord(); schema.setFields(fields); String schemaString = schema.toString(); assertNotNull(schemaString); assertEquals(2, schema.getFields().size()); }
private static Schema makePairSchema(Schema key, Schema value) { Schema pair = Schema.createRecord(PAIR, null, null, false); List<Field> fields = new ArrayList<>(); fields.add(new Field(KEY, key, "", null)); fields.add(new Field(VALUE, value, "", null, Field.Order.IGNORE)); pair.setFields(fields); return pair; }
defaultValue = new DoubleNode(Double.valueOf(defaultValue.textValue())); Field f = new Field(fieldName, fieldSchema, fieldDoc, defaultValue, order); Iterator<String> i = field.fieldNames(); fields.add(f); result.setFields(fields); } else if (type.equals("enum")) { // enum JsonNode symbolsNode = schema.get("symbols");
@Override public Schema onSchemaProcessed(Schema rootSchema, Set<Schema> addressableRecords) { List<Schema> deltaTypes = new ArrayList<Schema>(addressableRecords.size() + 1); deltaTypes.add(rootSchema); deltaTypes.addAll(addressableRecords); Field deltaTypesField = new Field(DELTA, Schema.createUnion(deltaTypes), null, null); List<Field> deltaFields = new ArrayList<Field>(); deltaFields.add(deltaTypesField); Schema delta = Schema.createRecord(DELTA + "T", null, KAA_NAMESPACE, false); delta.setFields(deltaFields); return Schema.createArray(delta); }
defaultValue = new DoubleNode(Double.valueOf(defaultValue.getTextValue())); Field f = new Field(fieldName, fieldSchema, fieldDoc, defaultValue, order); Iterator<String> i = field.getFieldNames(); fields.add(f); result.setFields(fields); } else if (type.equals("enum")) { // enum JsonNode symbolsNode = schema.get("symbols");
/** * Creates a KeyValuePair generic record schema. * * @return A schema for a generic record with two fields: 'key' and 'value'. */ public static Schema getSchema(Schema keySchema, Schema valueSchema) { Schema schema = Schema.createRecord( KEY_VALUE_PAIR_RECORD_NAME, "A key/value pair", KEY_VALUE_PAIR_RECORD_NAMESPACE, false); schema.setFields(Arrays.asList( new Schema.Field(KEY_FIELD, keySchema, "The key", null), new Schema.Field(VALUE_FIELD, valueSchema, "The value", null))); return schema; }
/** * Converts Hive schema to avro schema * * @param columnNames Names of the hive columns * @param columnTypes Hive Column types * @param namespace Namespace of Avro schema * @param name Avro schema name * @param doc Avro schema doc * @return Avro Schema */ public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, List<String> columnComments, String namespace, String name, String doc) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (int i = 0; i < columnNames.size(); ++i) { final String comment = columnComments.size() > i ? columnComments.get(i) : null; final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), comment); fields.addAll(getFields(avroField)); } if (name == null || name.isEmpty()) { name = "baseRecord"; } Schema avroSchema = Schema.createRecord(name, doc, namespace, false); avroSchema.setFields(fields); return avroSchema; }
/** * TODO: Currently generates a static schema avro record. **/ public static GenericRecord generateRandomAvroRecord() { ArrayList<Schema.Field> fields = new ArrayList<Schema.Field>(); String fieldName = "field1"; Schema fieldSchema = Schema.create(Schema.Type.STRING); String docString = "doc"; fields.add(new Schema.Field(fieldName, fieldSchema, docString, null)); Schema schema = Schema.createRecord("name", docString, "test",false); schema.setFields(fields); GenericData.Record record = new GenericData.Record(schema); record.put("field1", "foobar"); return record; }
/** * Creates a record with the given name, error status, and fields. * @param name the name of the schema. * @param isError true if the schema represents an error; false otherwise. * @param fields the field(s) to add to the schema. * @return the schema. */ private Schema createRecord(String name, boolean isError, Field... fields) { Schema record = Schema.createRecord(name, null, null, isError); record.setFields(Arrays.asList(fields)); return record; }
private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Set<Schema> processed) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); if (processed.contains(record)) { return Optional.absent(); } processed.add(record); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false); newSchema.setFields(fields); return Optional.of(newSchema); }
@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { Preconditions.checkArgument(inputSchema.getFields().equals(gobblinTrackingEventSchema.getFields())); Schema outputSchema = Schema .createRecord(ConfigUtils.getString(config, NEW_SCHEMA_NAME, inputSchema.getName()), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(newFields); return outputSchema; }
public static Optional<Schema> getKeySchemaFromRecord(Schema record) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = getKeySchema(field); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } if (!fields.isEmpty()) { Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getName(), false); newSchema.setFields(fields); return Optional.of(newSchema); } else { return Optional.absent(); } }
@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { List<Field> outputSchemaFields = new ArrayList<>(); for (Field field : inputSchema.getFields()) { outputSchemaFields.add(convertFieldSchema(inputSchema, field, workUnit)); } Schema outputSchema = Schema .createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(outputSchemaFields); return outputSchema; }
private Schema createAvroRecord(TypeInfo typeInfo) { List<Schema.Field> childFields = new ArrayList<Schema.Field>(); final List<String> allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); final List<TypeInfo> allStructFieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + "name and column type differs. names = " + allStructFieldNames + ", types = " + allStructFieldTypeInfos); } for (int i = 0; i < allStructFieldNames.size(); ++i) { final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), childTypeInfo, childTypeInfo.toString()); final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); childFields.addAll(grandChildFields); } Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), null, false); ++recordCounter; recordSchema.setFields(childFields); return recordSchema; }
schema = Schema.createEnum(c.getName(), null, null, symbols); } else if (TBase.class.isAssignableFrom(c)) { // struct schema = Schema.createRecord(c.getName(), null, null, Throwable.class.isAssignableFrom(c)); List<Field> fields = new ArrayList<>(); && (s.getType() != Schema.Type.UNION)) s = nullable(s); fields.add(new Field(f.fieldName, s, null, null)); schema.setFields(fields); } else { throw new RuntimeException("Not a Thrift-generated class: "+c);