TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
TableSchema schema = new TableSchema(); List<TableFieldSchema> tableFieldSchema = new ArrayList<TableFieldSchema>(); TableFieldSchema schemaEntry = new TableFieldSchema(); schemaEntry.setName(myFirstFieldName); schemaEntry.setType("STRING"); tableFieldSchema.add(schemaEntry); schema.setFields(tableFieldSchema); Table table = new Table(); table.setSchema(schema); TableReference tableRef = new TableReference(); tableRef.setDatasetId(DATASET_ID); tableRef.setProjectId(PROJECT_ID); tableRef.setTableId(tableId); table.setTableReference(tableRef); try { bigquery.tables().insert(PROJECT_ID, DATASET_ID, table).execute(); } catch (IOException e) { }
/** @see TableFieldSchema#setType(String) */ public BigQueryTableFieldSchema setType(String type) { fieldSchema.setType(type); return this; }
private TableFieldSchema tryFieldSchema(TableFieldSchema fieldSchema, org.apache.avro.Schema avroSchema) { fieldSchema = fieldSchema.setType(getBQFieldType(avroSchema)); if (avroSchema.getType() == org.apache.avro.Schema.Type.RECORD) { List<TableFieldSchema> childFields = new ArrayList<>(); List<org.apache.avro.Schema.Field> avroChildFields = avroSchema.getFields(); for (org.apache.avro.Schema.Field avroChildField : avroChildFields) { childFields.add(tryArrayFieldSchema(avroChildField)); } fieldSchema.setFields(childFields); } return fieldSchema; }
private TableFieldSchema tryFieldSchema(TableFieldSchema fieldSchema, org.apache.avro.Schema avroSchema) { fieldSchema = fieldSchema.setType(getBQFieldType(avroSchema)); if (avroSchema.getType() == org.apache.avro.Schema.Type.RECORD) { List<TableFieldSchema> childFields = new ArrayList<>(); List<org.apache.avro.Schema.Field> avroChildFields = avroSchema.getFields(); for (org.apache.avro.Schema.Field avroChildField : avroChildFields) { childFields.add(tryArrayFieldSchema(avroChildField)); } fieldSchema.setFields(childFields); } return fieldSchema; }
/** Helper method to build the table schema for the output table. */ private static TableSchema buildWeatherSchemaProjection() { List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("year").setType("INTEGER")); fields.add(new TableFieldSchema().setName("month").setType("INTEGER")); fields.add(new TableFieldSchema().setName("day").setType("INTEGER")); fields.add(new TableFieldSchema().setName("mean_temp").setType("FLOAT")); TableSchema schema = new TableSchema().setFields(fields); return schema; }
/** Defines the BigQuery schema used for the output. */ static TableSchema getSchema() { List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("route").setType("STRING")); fields.add(new TableFieldSchema().setName("avg_speed").setType("FLOAT")); fields.add(new TableFieldSchema().setName("slowdown_event").setType("BOOLEAN")); fields.add(new TableFieldSchema().setName("window_timestamp").setType("TIMESTAMP")); TableSchema schema = new TableSchema().setFields(fields); return schema; } }
/** Defines the BigQuery schema used for the output. */ private static TableSchema getSchema() { List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("trigger_type").setType("STRING")); fields.add(new TableFieldSchema().setName("freeway").setType("STRING")); fields.add(new TableFieldSchema().setName("total_flow").setType("INTEGER")); fields.add(new TableFieldSchema().setName("number_of_records").setType("INTEGER")); fields.add(new TableFieldSchema().setName("window").setType("STRING")); fields.add(new TableFieldSchema().setName("isFirst").setType("BOOLEAN")); fields.add(new TableFieldSchema().setName("isLast").setType("BOOLEAN")); fields.add(new TableFieldSchema().setName("timing").setType("STRING")); fields.add(new TableFieldSchema().setName("event_time").setType("TIMESTAMP")); fields.add(new TableFieldSchema().setName("processing_time").setType("TIMESTAMP")); TableSchema schema = new TableSchema().setFields(fields); return schema; }
/** Build the output table schema. */ protected TableSchema getSchema() { List<TableFieldSchema> fields = new ArrayList<>(); for (Map.Entry<String, FieldInfo<InputT>> entry : fieldInfo.entrySet()) { String key = entry.getKey(); FieldInfo<InputT> fcnInfo = entry.getValue(); String bqType = fcnInfo.getFieldType(); fields.add(new TableFieldSchema().setName(key).setType(bqType)); } return new TableSchema().setFields(fields); }
/** Defines the BigQuery schema used for the output. */ static TableSchema getSchema() { List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("station_id").setType("STRING")); fields.add(new TableFieldSchema().setName("direction").setType("STRING")); fields.add(new TableFieldSchema().setName("freeway").setType("STRING")); fields.add(new TableFieldSchema().setName("lane_max_flow").setType("INTEGER")); fields.add(new TableFieldSchema().setName("lane").setType("STRING")); fields.add(new TableFieldSchema().setName("avg_occ").setType("FLOAT")); fields.add(new TableFieldSchema().setName("avg_speed").setType("FLOAT")); fields.add(new TableFieldSchema().setName("total_flow").setType("INTEGER")); fields.add(new TableFieldSchema().setName("window_timestamp").setType("TIMESTAMP")); fields.add(new TableFieldSchema().setName("recorded_timestamp").setType("STRING")); TableSchema schema = new TableSchema().setFields(fields); return schema; } }
/** Defines the BigQuery schema used for the output. */ static TableSchema getSchema() { List<TableFieldSchema> tagFields = new ArrayList<>(); tagFields.add(new TableFieldSchema().setName("count").setType("INTEGER")); tagFields.add(new TableFieldSchema().setName("tag").setType("STRING")); List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("prefix").setType("STRING")); fields.add( new TableFieldSchema() .setName("tags") .setType("RECORD") .setMode("REPEATED") .setFields(tagFields)); return new TableSchema().setFields(fields); } }
static TableSchema getSchema() { return new TableSchema() .setFields( Collections.singletonList( new TableFieldSchema().setName("string_field").setType("STRING"))); } }
private List<TableFieldSchema> langStringFields(PropertyConstraint p) { List<TableFieldSchema> list = new ArrayList<>(); TableFieldSchema stringValue = new TableFieldSchema(); list.add(stringValue); stringValue.setName("stringValue"); stringValue.setMode(FieldMode.REQUIRED.name()); stringValue.setType(BigQueryDatatype.STRING.name()); TableFieldSchema languageCode = new TableFieldSchema(); list.add(languageCode); languageCode.setName("languageCode"); languageCode.setMode(FieldMode.REQUIRED.name()); languageCode.setType(BigQueryDatatype.STRING.name()); return list; }
static TableFieldSchema getPositionFieldSchema(String fieldName) { TableFieldSchema fieldSchema = new TableFieldSchema().setName(fieldName).setType(RECORD); fieldSchema.setFields(new ArrayList<TableFieldSchema>() { private static final long serialVersionUID = 0; { add(new TableFieldSchema().setName(REFERENCE_NAME).setType(STRING)); add(new TableFieldSchema().setName(POSITION).setType(INT64)); add(new TableFieldSchema().setName(REVERSE_STRAND).setType(BOOL)); } }); return fieldSchema; }
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (getSubFields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(getSubFields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (fields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
TableFieldSchema toPb() { TableFieldSchema fieldSchemaPb = new TableFieldSchema(); fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); } if (fields() != null) { List<TableFieldSchema> fieldsPb = Lists.transform(fields(), TO_PB_FUNCTION); fieldSchemaPb.setFields(fieldsPb); } return fieldSchemaPb; }
static void runBigQueryTornadoes(Options options) { Pipeline p = Pipeline.create(options); // Build the table schema for the output table. List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("month").setType("INTEGER")); fields.add(new TableFieldSchema().setName("tornado_count").setType("INTEGER")); TableSchema schema = new TableSchema().setFields(fields); p.apply(BigQueryIO.readTableRows().from(options.getInput())) .apply(new CountTornadoes()) .apply( BigQueryIO.writeTableRows() .to(options.getOutput()) .withSchema(schema) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)); p.run().waitUntilFinish(); }