public static Field mapToField(Map fMap) { String typeStr = fMap.get("type").toString(); String nameStr = fMap.get("name").toString(); String modeStr = fMap.get("mode").toString(); LegacySQLTypeName type = null; if (typeStr.equals("BOOLEAN")) { type = LegacySQLTypeName.BOOLEAN; } else if (typeStr.equals("STRING")) { type = LegacySQLTypeName.STRING; } else if (typeStr.equals("BYTES")) { type = LegacySQLTypeName.BYTES; } else if (typeStr.equals("INTEGER")) { type = LegacySQLTypeName.INTEGER; } else if (typeStr.equals("FLOAT")) { type = LegacySQLTypeName.FLOAT; } else if (typeStr.equals("TIMESTAMP") || typeStr.equals("DATE") || typeStr.equals("TIME") || typeStr.equals("DATETIME")) { type = LegacySQLTypeName.TIMESTAMP; } else if (typeStr.equals("RECORD")) { type = LegacySQLTypeName.RECORD; } return Field.newBuilder(nameStr, type).setMode(Field.Mode.valueOf(modeStr)).build(); }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
Schema.of( Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(), Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build(), Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build()); assertEquals(expectedSchema, remoteTable.getDefinition().getSchema());
@Test public void testDecimal() { final String fieldName = "Decimal"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.FLOAT ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Decimal.schema(0)) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testDate() { final String fieldName = "Date"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.DATE ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Date.SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testString() { final String fieldName = "String"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.STRING_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testTimestamp() { final String fieldName = "Timestamp"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.TIMESTAMP ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Timestamp.SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testBytes() { final String fieldName = "Bytes"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.BYTES ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.BYTES_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testDescription() { final String fieldName = "WithDoc"; final String fieldDoc = "test documentation"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.STRING) .setMode(com.google.cloud.bigquery.Field.Mode.REQUIRED) .setDescription(fieldDoc) .build() ); Schema kafkaConnectTestSchema = SchemaBuilder.struct() .field(fieldName, SchemaBuilder.string().doc(fieldDoc).build()) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testBoolean() { final String fieldName = "Boolean"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.BOOLEAN ).setMode( com.google.cloud.bigquery.Field.Mode.REQUIRED ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, Schema.BOOLEAN_SCHEMA) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testStringArray() { final String fieldName = "StringArray"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ); Schema kafkaConnectArraySchema = SchemaBuilder.array(Schema.STRING_SCHEMA).build(); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, kafkaConnectArraySchema) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testIntegerArray() { final String fieldName = "IntegerArray"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.INTEGER ).setMode(com.google.cloud.bigquery.Field.Mode.REPEATED).build() ); Schema kafkaConnectArraySchema = SchemaBuilder.array(Schema.INT32_SCHEMA).build(); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, kafkaConnectArraySchema) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); }
@Test public void testAllFieldsNullable() { final String fieldName = "RequiredField"; com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of( com.google.cloud.bigquery.Field.newBuilder( fieldName, LegacySQLTypeName.STRING ).setMode( com.google.cloud.bigquery.Field.Mode.NULLABLE ).build() ); Schema kafkaConnectTestSchema = SchemaBuilder .struct() .field(fieldName, SchemaBuilder.string().required().build()) .build(); com.google.cloud.bigquery.Schema bigQueryTestSchema = new BigQuerySchemaConverter(true).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryTestSchema); } }
private Field getKafkaDataField() { Field topicField = Field.of("topic", LegacySQLTypeName.STRING); Field partitionField = Field.of("partition", LegacySQLTypeName.INTEGER); Field offsetField = Field.of("offset", LegacySQLTypeName.INTEGER); Field insertTimeField = Field.newBuilder("insertTime",LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(); return Field.newBuilder("kafkaData", LegacySQLTypeName.RECORD, topicField, partitionField, offsetField, insertTimeField) .setMode(Field.Mode.NULLABLE) .build(); } }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
private com.google.cloud.bigquery.Field.Builder convertMap(Schema kafkaConnectSchema, String fieldName) { Schema keySchema = kafkaConnectSchema.keySchema(); Schema valueSchema = kafkaConnectSchema.valueSchema(); com.google.cloud.bigquery.Field keyField = convertField(keySchema, MAP_KEY_FIELD_NAME).build(); com.google.cloud.bigquery.Field valueField = convertField(valueSchema, MAP_VALUE_FIELD_NAME).build(); com.google.cloud.bigquery.Field.Builder bigQueryRecordBuilder = com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.RECORD, keyField, valueField); return bigQueryRecordBuilder.setMode(com.google.cloud.bigquery.Field.Mode.REPEATED); }
private void setNullability(Schema kafkaConnectSchema, com.google.cloud.bigquery.Field.Builder fieldBuilder) { switch (kafkaConnectSchema.type()) { case ARRAY: case MAP: return; default: if (allFieldsNullable || kafkaConnectSchema.isOptional()) { fieldBuilder.setMode(com.google.cloud.bigquery.Field.Mode.NULLABLE); } else { fieldBuilder.setMode(com.google.cloud.bigquery.Field.Mode.REQUIRED); } } }
private static Field field(TableFieldSchema field) { return Field.newBuilder(field.getName(), fieldType(field)).setMode(mode(field)).build(); }
@Test public void test() { Schema kafkaConnectTestSchema = SchemaBuilder.struct().field("base", Schema.STRING_SCHEMA).build(); Field kafkaDataField = getKafkaDataField(); Field baseField = Field.newBuilder("base", LegacySQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(); com.google.cloud.bigquery.Schema bigQueryExpectedSchema = com.google.cloud.bigquery.Schema.of(baseField, kafkaDataField); com.google.cloud.bigquery.Schema bigQueryActualSchema = new KafkaDataBQSchemaConverter(false).convertSchema(kafkaConnectTestSchema); assertEquals(bigQueryExpectedSchema, bigQueryActualSchema); }
private com.google.cloud.bigquery.Field.Builder convertArray(Schema kafkaConnectSchema, String fieldName) { Schema elementSchema = kafkaConnectSchema.valueSchema(); com.google.cloud.bigquery.Field.Builder elementFieldBuilder = convertField(elementSchema, fieldName); return elementFieldBuilder.setMode(com.google.cloud.bigquery.Field.Mode.REPEATED); }