/** Returns a Field object with given name and type. */ public static Field of(String name, LegacySQLTypeName type, FieldList subFields) { return newBuilder(name, type, subFields).build(); }
/** Example of listing table rows with schema. */ // [TARGET listTableData(TableId, Schema, TableDataListOption...)] public FieldValueList listTableDataSchemaId() { // [START ] Schema schema = Schema.of( Field.of("word", LegacySQLTypeName.STRING), Field.of("word_count", LegacySQLTypeName.STRING), Field.of("corpus", LegacySQLTypeName.STRING), Field.of("corpus_date", LegacySQLTypeName.STRING)); TableResult tableData = bigquery.listTableData( TableId.of("bigquery-public-data", "samples", "shakespeare"), schema); FieldValueList row = tableData.getValues().iterator().next(); System.out.println(row.get("word").getStringValue()); // [END ] return row; }
private void compareFieldSchemas(Field expected, Field value) { assertEquals(expected, value); assertEquals(expected.getName(), value.getName()); assertEquals(expected.getType(), value.getType()); assertEquals(expected.getMode(), value.getMode()); assertEquals(expected.getDescription(), value.getDescription()); assertEquals(expected.getSubFields(), value.getSubFields()); } }
@Test public void testToAndFromPb() { compareFieldSchemas(FIELD_SCHEMA1, Field.fromPb(FIELD_SCHEMA1.toPb())); compareFieldSchemas(FIELD_SCHEMA2, Field.fromPb(FIELD_SCHEMA2.toPb())); compareFieldSchemas(FIELD_SCHEMA3, Field.fromPb(FIELD_SCHEMA3.toPb())); Field field = Field.newBuilder(FIELD_NAME1, FIELD_TYPE1).build(); compareFieldSchemas(field, Field.fromPb(field.toPb())); }
super.convertSchema(kafkaConnectSchema); Field topicField = Field.of(KAFKA_DATA_TOPIC_FIELD_NAME, LegacySQLTypeName.STRING); Field partitionField = Field.of(KAFKA_DATA_PARTITION_FIELD_NAME, LegacySQLTypeName.INTEGER); Field offsetField = Field.of(KAFKA_DATA_OFFSET_FIELD_NAME, LegacySQLTypeName.INTEGER); Field.Builder insertTimeBuilder = Field.newBuilder(KAFKA_DATA_INSERT_TIME_FIELD_NAME, LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE); Field.newBuilder(KAFKA_DATA_FIELD_NAME, LegacySQLTypeName.RECORD, topicField, partitionField, offsetField,
private org.apache.avro.Schema inferSchemaField(Field field) { String name = field.getName(); Field.Type sqlType = field.getType(); Field.Mode mode = field.getMode(); // Get the "basic" type of the field. org.apache.avro.Schema fieldSchema = inferSchemaFieldWithoutMode(field); // BigQuery fields are NULLABLE by default. if (Field.Mode.NULLABLE == mode || mode == null) { fieldSchema = AvroUtils.wrapAsNullable(fieldSchema); } else if (Field.Mode.REPEATED == mode) { // Determine if the field is an array. // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type fieldSchema = SchemaBuilder.array().items(fieldSchema); } return fieldSchema; }
private FieldList(Iterable<Field> fields) { this.fields = ImmutableList.copyOf(fields); ImmutableMap.Builder<String, Integer> nameIndexBuilder = ImmutableMap.builder(); int index = 0; for (Field field : fields) { nameIndexBuilder.put(field.getName(), index); index++; } this.nameIndex = nameIndexBuilder.build(); }
@Override public TableFieldSchema apply(Field field) { return field.toPb(); } };
@Override public Field apply(TableFieldSchema pb) { return Field.fromPb(pb); } };
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
/** Creates a {@code Field} object. */ public Field build() { return new Field(this); } }
private Field getKafkaDataField() { Field topicField = Field.of("topic", LegacySQLTypeName.STRING); Field partitionField = Field.of("partition", LegacySQLTypeName.INTEGER); Field offsetField = Field.of("offset", LegacySQLTypeName.INTEGER); Field insertTimeField = Field.newBuilder("insertTime",LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(); return Field.newBuilder("kafkaData", LegacySQLTypeName.RECORD, topicField, partitionField, offsetField, insertTimeField) .setMode(Field.Mode.NULLABLE) .build(); } }
private org.apache.avro.Schema inferSchemaField(Field field) { String name = field.getName(); Field.Type sqlType = field.getType(); Field.Mode mode = field.getMode(); // Get the "basic" type of the field. org.apache.avro.Schema fieldSchema = inferSchemaFieldWithoutMode(field); // BigQuery fields are NULLABLE by default. if (Field.Mode.NULLABLE == mode || mode == null) { fieldSchema = AvroUtils.wrapAsNullable(fieldSchema); } else if (Field.Mode.REPEATED == mode) { // Determine if the field is an array. // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type fieldSchema = SchemaBuilder.array().items(fieldSchema); } return fieldSchema; }
TimePartitioning partitioning = TimePartitioning.of(Type.DAY); Clustering clustering = Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); StandardTableDefinition tableDefinition = StandardTableDefinition.newBuilder()
@Override public Field apply(TableFieldSchema pb) { return Field.fromPb(pb); } };
@SuppressWarnings("unchecked") static FieldValue fromPb(Object cellPb, Field recordSchema) { if (Data.isNull(cellPb)) { return FieldValue.of(Attribute.PRIMITIVE, null); } if (cellPb instanceof String) { return FieldValue.of(Attribute.PRIMITIVE, cellPb); } if (cellPb instanceof List) { return FieldValue.of(Attribute.REPEATED, FieldValueList.fromPb((List<Object>) cellPb, null)); } if (cellPb instanceof Map) { Map<String, Object> cellMapPb = (Map<String, Object>) cellPb; if (cellMapPb.containsKey("f")) { FieldList subFieldsSchema = recordSchema != null ? recordSchema.getSubFields() : null; return FieldValue.of( Attribute.RECORD, FieldValueList.fromPb((List<Object>) cellMapPb.get("f"), subFieldsSchema)); } // This should never be the case when we are processing a first level table field (i.e. a // row's field, not a record sub-field) if (cellMapPb.containsKey("v")) { return FieldValue.fromPb(cellMapPb.get("v"), recordSchema); } } throw new IllegalArgumentException("Unexpected table cell format"); } }