public static Field mapToField(Map fMap) { String typeStr = fMap.get("type").toString(); String nameStr = fMap.get("name").toString(); String modeStr = fMap.get("mode").toString(); LegacySQLTypeName type = null; if (typeStr.equals("BOOLEAN")) { type = LegacySQLTypeName.BOOLEAN; } else if (typeStr.equals("STRING")) { type = LegacySQLTypeName.STRING; } else if (typeStr.equals("BYTES")) { type = LegacySQLTypeName.BYTES; } else if (typeStr.equals("INTEGER")) { type = LegacySQLTypeName.INTEGER; } else if (typeStr.equals("FLOAT")) { type = LegacySQLTypeName.FLOAT; } else if (typeStr.equals("TIMESTAMP") || typeStr.equals("DATE") || typeStr.equals("TIME") || typeStr.equals("DATETIME")) { type = LegacySQLTypeName.TIMESTAMP; } else if (typeStr.equals("RECORD")) { type = LegacySQLTypeName.RECORD; } return Field.newBuilder(nameStr, type).setMode(Field.Mode.valueOf(modeStr)).build(); }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
innerFieldStringName, LegacySQLTypeName.STRING ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build(), com.google.cloud.bigquery.Field.newBuilder( innerFieldIntegerName, LegacySQLTypeName.INTEGER ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build(); middleFieldArrayName, LegacySQLTypeName.FLOAT ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build(); bigQueryInnerRecord, bigQueryMiddleRecord ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build();
result.setDescription(kafkaConnectSchema.doc());
@Test public void testFloat() { final String fieldName = "Float"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.FLOAT ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.FLOAT32_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.FLOAT64_SCHEMA) .build(); bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testNullable() { final String nullableFieldName = "Nullable"; final String requiredFieldName = "Required"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( nullableFieldName, LegacySQLTypeName.INTEGER ).setMode( com.google.cloud.bigquery.Field.Mode.NULLABLE ).build(), com.google.cloud.bigquery.Field.newBuilder( requiredFieldName, LegacySQLTypeName.INTEGER ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(nullableFieldName, SchemaBuilder.int32().optional().build()) .field(requiredFieldName, SchemaBuilder.int32().required().build()) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
/** * Convert a {@link Schema Kafka Connect Schema} into a * {@link com.google.cloud.bigquery.Schema BigQuery schema}. * * @param kafkaConnectSchema The schema to convert. Must be of type Struct, in order to translate * into a row format that requires each field to consist of both a name * and a value. * @return The resulting schema, which can then be used to create a new table or update an * existing one. */ public com.google.cloud.bigquery.Schema convertSchema(Schema kafkaConnectSchema) { if (kafkaConnectSchema.type() != Schema.Type.STRUCT) { throw new ConversionConnectException("Top-level Kafka Connect schema must be of type 'struct'"); } List<com.google.cloud.bigquery.Field> fields = new LinkedList<>(); for (Field kafkaConnectField : kafkaConnectSchema.fields()) { com.google.cloud.bigquery.Field bigQuerySchemaField = convertField(kafkaConnectField.schema(), kafkaConnectField.name()).build(); fields.add(bigQuerySchemaField); } return com.google.cloud.bigquery.Schema.of(fields); }
@Test public void testDecimal() { final String fieldName = "Decimal"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.FLOAT ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Decimal.schema(0)) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testDate() { final String fieldName = "Date"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.DATE ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Date.SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testString() { final String fieldName = "String"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.STRING_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testTimestamp() { final String fieldName = "Timestamp"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.TIMESTAMP ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Timestamp.SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testBytes() { final String fieldName = "Bytes"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.BYTES ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.BYTES_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testBoolean() { final String fieldName = "Boolean"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.BOOLEAN ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.BOOLEAN_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testAllFieldsNullable() { final String fieldName = "RequiredField"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode( com.google.cloud.bigquery.Field.Mode.NULLABLE ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, SchemaBuilder.string().required().build()) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(true).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); } }
@Test public void testDescription() { final String fieldName = "WithDoc"; final String fieldDoc = "test documentation"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.STRING) .setMode(com.google.cloud.bigquery.Field.Mode.REQUIRED) .setDescription(fieldDoc) .build() ); Schema kafkaConnectTestSchema = SchemaBuilder.struct() .field(fieldName, SchemaBuilder.string().doc(fieldDoc).build()) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testStringArray() { final String fieldName = "StringArray"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ); Schema kafkaConnectArraySchema = SchemaBuilder.array(Schema.STRING_SCHEMA).build(); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, kafkaConnectArraySchema) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testIntegerArray() { final String fieldName = "IntegerArray"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.INTEGER ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ); Schema kafkaConnectArraySchema = SchemaBuilder.array(Schema.INT32_SCHEMA).build(); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, kafkaConnectArraySchema) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }