Refine search
/** Create an anonymous record schema. */ public static Schema createRecord(List<Field> fields) { Schema result = createRecord(null, null, null, false); result.setFields(fields); return result; }
private static Schema _getDeleteSchema() { List<Schema.Field> schemaFields = new ArrayList<>(1); Schema.Field designField = new Schema.Field( AvroConstants.ID, AvroUtils._string(), null, (Object)null); designField.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); schemaFields.add(designField); Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields); return schema; }
/** Create an anonymous record schema. */ public static Schema createRecord(List<Field> fields) { Schema result = createRecord(null, null, null, false); result.setFields(fields); return result; }
Schema.Field designField = new Schema.Field( fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); return Schema.createRecord("Runtime", null, null, false, schemaFields);
/** * Creates a KeyValuePair generic record schema. * * @return A schema for a generic record with two fields: 'key' and * 'value'. */ public static Schema getSchema(Schema keySchema, Schema valueSchema) { Schema schema = Schema.createRecord(KEY_VALUE_PAIR_RECORD_NAME, "A key/value pair", KEY_VALUE_PAIR_RECORD_NAMESPACE, false); schema.setFields(Arrays.asList(new Schema.Field(KEY_FIELD, keySchema, "The key", null), new Schema.Field(VALUE_FIELD, valueSchema, "The value", null))); return schema; } }
private void _setupSchemas() { Schema.Field docIdField = new Schema.Field( "resourceId", AvroUtils._string(), null, (Object)null, Schema.Field.Order.ASCENDING); docIdField.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true"); docIdField.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); List<Schema.Field> fields = new ArrayList<>(); fields.add(docIdField); Schema initialSchema = Schema.createRecord( "liferay", null, null, false, fields); resource.main.schema.setValue(initialSchema); _updateOutputSchemas(); }
private static Schema makePairSchema(Schema key, Schema value) { Schema pair = Schema.createRecord(PAIR, null, null, false); List<Field> fields = new ArrayList<>(); fields.add(new Field(KEY, key, "", null)); fields.add(new Field(VALUE, value, "", null, Field.Order.IGNORE)); pair.setFields(fields); return pair; }
Schema.Field designField = new Schema.Field( fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields);
@Override public Schema onSchemaProcessed(Schema rootSchema, Set<Schema> addressableRecords) { List<Schema> deltaTypes = new ArrayList<Schema>(addressableRecords.size() + 1); deltaTypes.add(rootSchema); deltaTypes.addAll(addressableRecords); Field deltaTypesField = new Field(DELTA, Schema.createUnion(deltaTypes), null, null); List<Field> deltaFields = new ArrayList<Field>(); deltaFields.add(deltaTypesField); Schema delta = Schema.createRecord(DELTA + "T", null, KAA_NAMESPACE, false); delta.setFields(deltaFields); return Schema.createArray(delta); }
public static Schema inferSchemaByResourceType( ApioApiDocumentation.SupportedClass resourceSupportedClass) { List<Property> supportedProperties = resourceSupportedClass.getSupportedProperties(); List<Schema.Field> schemaFields = new ArrayList<>( supportedProperties.size() + 1); // Already used names for the fields Set<String> fieldNames = new HashSet<>(); _addIdSchemaField(schemaFields, fieldNames); int i = 1; for (Property supportedProperty : supportedProperties) { String fieldName = NameUtil.correct( supportedProperty.getName(), i, fieldNames); fieldNames.add(fieldName); Schema.Field designField = new Schema.Field( fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); schemaFields.add(i, designField); i++; } Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields); return schema; }
/** * Creates a KeyValuePair generic record schema. * * @return A schema for a generic record with two fields: 'key' and 'value'. */ public static Schema getSchema(Schema keySchema, Schema valueSchema) { Schema schema = Schema.createRecord( KEY_VALUE_PAIR_RECORD_NAME, "A key/value pair", KEY_VALUE_PAIR_RECORD_NAMESPACE, false); schema.setFields(Arrays.asList( new Schema.Field(KEY_FIELD, keySchema, "The key", null), new Schema.Field(VALUE_FIELD, valueSchema, "The value", null))); return schema; }
? paramNames[i] : paramSchema.getName()+i; fields.add(new Schema.Field(paramName, paramSchema, Schema request = Schema.createRecord(fields);
/** * Converts Hive schema to avro schema * * @param columnNames Names of the hive columns * @param columnTypes Hive Column types * @param namespace Namespace of Avro schema * @param name Avro schema name * @param doc Avro schema doc * @return Avro Schema */ public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, List<String> columnComments, String namespace, String name, String doc) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (int i = 0; i < columnNames.size(); ++i) { final String comment = columnComments.size() > i ? columnComments.get(i) : null; final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), comment); fields.addAll(getFields(avroField)); } if (name == null || name.isEmpty()) { name = "baseRecord"; } Schema avroSchema = Schema.createRecord(name, doc, namespace, false); avroSchema.setFields(fields); return avroSchema; }
? paramNames[i] : paramSchema.getName()+i; fields.add(new Schema.Field(paramName, paramSchema, Schema request = Schema.createRecord(fields);
/** * TODO: Currently generates a static schema avro record. **/ public static GenericRecord generateRandomAvroRecord() { ArrayList<Schema.Field> fields = new ArrayList<Schema.Field>(); String fieldName = "field1"; Schema fieldSchema = Schema.create(Schema.Type.STRING); String docString = "doc"; fields.add(new Schema.Field(fieldName, fieldSchema, docString, null)); Schema schema = Schema.createRecord("name", docString, "test",false); schema.setFields(fields); GenericData.Record record = new GenericData.Record(schema); record.put("field1", "foobar"); return record; }
@Test public void testIncomingTimeColumn() throws Exception { List<Schema.Field> avroFields = Collections.singletonList(new Schema.Field("incomingTime", Schema.create(Schema.Type.LONG), null, null)); Schema avroSchema = Schema.createRecord(avroFields); GenericData.Record avroRecord = new GenericData.Record(avroSchema); avroRecord.put("incomingTime", 12345L); org.apache.pinot.common.data.Schema pinotSchema = new org.apache.pinot.common.data.Schema.SchemaBuilder().setSchemaName("testSchema") .addTime("incomingTime", TimeUnit.MILLISECONDS, FieldSpec.DataType.LONG, "outgoingTime", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); AvroRecordToPinotRowGenerator avroRecordToPinotRowGenerator = new AvroRecordToPinotRowGenerator(pinotSchema); GenericRow genericRow = new GenericRow(); avroRecordToPinotRowGenerator.transform(avroRecord, genericRow); Assert.assertEquals(genericRow.getFieldNames(), new String[]{"incomingTime"}); Assert.assertEquals(genericRow.getValue("incomingTime"), 12345L); } }
/** * Creates a record with the given name, error status, and fields. * @param name the name of the schema. * @param isError true if the schema represents an error; false otherwise. * @param fields the field(s) to add to the schema. * @return the schema. */ private Schema createRecord(String name, boolean isError, Field... fields) { Schema record = Schema.createRecord(name, null, null, isError); record.setFields(Arrays.asList(fields)); return record; }
@Test public void testValidateSchemaPairMissingField() throws Exception { final List<Schema.Field> readerFields = list( new Schema.Field("oldfield1", INT_SCHEMA, null, null)); final Schema reader = Schema.createRecord(readerFields); final SchemaCompatibility.SchemaPairCompatibility expectedResult = new SchemaCompatibility.SchemaPairCompatibility( SchemaCompatibility.SchemaCompatibilityResult.compatible(), reader, WRITER_SCHEMA, SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE); // Test omitting a field. assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA)); }
private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Set<Schema> processed) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); if (processed.contains(record)) { return Optional.absent(); } processed.add(record); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false); newSchema.setFields(fields); return Optional.of(newSchema); }
@Test public void testValidateSchemaPairMissingSecondField() throws Exception { final List<Schema.Field> readerFields = list( new Schema.Field("oldfield2", STRING_SCHEMA, null, null)); final Schema reader = Schema.createRecord(readerFields); final SchemaCompatibility.SchemaPairCompatibility expectedResult = new SchemaCompatibility.SchemaPairCompatibility( SchemaCompatibility.SchemaCompatibilityResult.compatible(), reader, WRITER_SCHEMA, SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE); // Test omitting other field. assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA)); }