/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, FieldList subFields) { return newBuilder(name, type, subFields).build(); }
/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, Field... subFields) { return newBuilder(name, type, subFields).build(); }
public static Field mapToField(Map fMap) { String typeStr = fMap.get("type").toString(); String nameStr = fMap.get("name").toString(); String modeStr = fMap.get("mode").toString(); LegacySQLTypeName type = null; if (typeStr.equals("BOOLEAN")) { type = LegacySQLTypeName.BOOLEAN; } else if (typeStr.equals("STRING")) { type = LegacySQLTypeName.STRING; } else if (typeStr.equals("BYTES")) { type = LegacySQLTypeName.BYTES; } else if (typeStr.equals("INTEGER")) { type = LegacySQLTypeName.INTEGER; } else if (typeStr.equals("FLOAT")) { type = LegacySQLTypeName.FLOAT; } else if (typeStr.equals("TIMESTAMP") || typeStr.equals("DATE") || typeStr.equals("TIME") || typeStr.equals("DATETIME")) { type = LegacySQLTypeName.TIMESTAMP; } else if (typeStr.equals("RECORD")) { type = LegacySQLTypeName.RECORD; } return Field.newBuilder(nameStr, type).setMode(Field.Mode.valueOf(modeStr)).build(); }
@Test public void testToAndFromPb() { compareFieldSchemas(FIELD_SCHEMA1, Field.fromPb(FIELD_SCHEMA1.toPb())); compareFieldSchemas(FIELD_SCHEMA2, Field.fromPb(FIELD_SCHEMA2.toPb())); compareFieldSchemas(FIELD_SCHEMA3, Field.fromPb(FIELD_SCHEMA3.toPb())); Field field = Field.newBuilder(FIELD_NAME1, FIELD_TYPE1).build(); compareFieldSchemas(field, Field.fromPb(field.toPb())); }
Schema expectedSchema = Schema.of( Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(), Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build(), Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build());
private com.google.cloud.bigquery.Field.Builder convertPrimitive(Schema kafkaConnectSchema, String fieldName) { LegacySQLTypeName bigQueryType = PRIMITIVE_TYPE_MAP.get(kafkaConnectSchema.type()); return com.google.cloud.bigquery.Field.newBuilder(fieldName, bigQueryType); }
/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, FieldList subFields) { return newBuilder(name, type, subFields).build(); }
/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, Field... subFields) { return newBuilder(name, type, subFields).build(); }
private Field createField(String name, Enum valueType, String description) { Field.Builder fieldBuilder = Field.newBuilder(name, FEAST_TO_BIGQUERY_TYPE_MAP.get(valueType)); if (description != null) { fieldBuilder.setDescription(description); } return fieldBuilder.build(); }
private com.google.cloud.bigquery.Field.Builder convertStruct(Schema kafkaConnectSchema, String fieldName) { List<com.google.cloud.bigquery.Field> bigQueryRecordFields = new ArrayList<>(); for (Field kafkaConnectField : kafkaConnectSchema.fields()) { com.google.cloud.bigquery.Field.Builder bigQueryRecordFieldBuilder = convertField(kafkaConnectField.schema(), kafkaConnectField.name()); bigQueryRecordFields.add(bigQueryRecordFieldBuilder.build()); } FieldList fieldList = FieldList.of(bigQueryRecordFields); return com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.RECORD, fieldList); }
private com.google.cloud.bigquery.Field.Builder convertLogical(Schema kafkaConnectSchema, String fieldName) { LogicalTypeConverter converter = LogicalConverterRegistry.getConverter(kafkaConnectSchema.name()); converter.checkEncodingType(kafkaConnectSchema.type()); return com.google.cloud.bigquery.Field.newBuilder(fieldName, converter.getBQSchemaType()); } }
private static Field field(TableFieldSchema field) { return Field.newBuilder(field.getName(), fieldType(field)).setMode(mode(field)).build(); }
private com.google.cloud.bigquery.Field.Builder convertMap(Schema kafkaConnectSchema, String fieldName) { Schema keySchema = kafkaConnectSchema.keySchema(); Schema valueSchema = kafkaConnectSchema.valueSchema(); com.google.cloud.bigquery.Field keyField = convertField(keySchema, MAP_KEY_FIELD_NAME).build(); com.google.cloud.bigquery.Field valueField = convertField(valueSchema, MAP_VALUE_FIELD_NAME).build(); com.google.cloud.bigquery.Field.Builder bigQueryRecordBuilder = com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.RECORD, keyField, valueField); return bigQueryRecordBuilder.setMode(com.google.cloud.bigquery.Field.Mode.REPEATED); }
private Field getKafkaDataField() { Field topicField = Field.of("topic", LegacySQLTypeName.STRING); Field partitionField = Field.of("partition", LegacySQLTypeName.INTEGER); Field offsetField = Field.of("offset", LegacySQLTypeName.INTEGER); Field insertTimeField = Field.newBuilder("insertTime",LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(); return Field.newBuilder("kafkaData", LegacySQLTypeName.RECORD, topicField, partitionField, offsetField, insertTimeField) .setMode(Field.Mode.NULLABLE) .build(); } }
@Test public void test() { Schema kafkaConnectTestSchema = SchemaBuilder.struct().field("base", Schema.STRING_SCHEMA).build(); Field kafkaDataField = getKafkaDataField(); Field baseField = Field.newBuilder("base", LegacySQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(); com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of(baseField, kafkaDataField); com.google.cloud.bigquery.Schema bigQueryActualSchema = new KafkaDataBQSchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryActualSchema); }
@Test public void testBytes() { final String fieldName = "Bytes"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.BYTES ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.BYTES_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testTimestamp() { final String fieldName = "Timestamp"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.TIMESTAMP ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Timestamp.SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testIntegerArray() { final String fieldName = "IntegerArray"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.INTEGER ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ); Schema kafkaConnectArraySchema = SchemaBuilder.array(Schema.INT32_SCHEMA).build(); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, kafkaConnectArraySchema) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testStringArray() { final String fieldName = "StringArray"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ); Schema kafkaConnectArraySchema = SchemaBuilder.array(Schema.STRING_SCHEMA).build(); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, kafkaConnectArraySchema) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testAllFieldsNullable() { final String fieldName = "RequiredField"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode( com.google.cloud.bigquery.Field.Mode.NULLABLE ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, SchemaBuilder.string().required().build()) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(true).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); } }