static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { List<TableFieldSchema> fields = tableSchemaPb.getFields(); if (fields == null) { fields = Collections.emptyList(); } return Schema.of(FieldList.fromPb(fields)); } }
/** * Utility function to convert from an Avro {@link GenericRecord} to a BigQuery {@link TableRow}. * * <p>See <a href="https://cloud.google.com/bigquery/exporting-data-from-bigquery#config">"Avro * format"</a> for more information. */ static TableRow convertGenericRecordToTableRow(GenericRecord record, TableSchema schema) { return convertGenericRecordToTableRow(record, schema.getFields()); }
static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { return Schema.of(Lists.transform(tableSchemaPb.getFields(), Field.FROM_PB_FUNCTION)); } }
static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { return Schema.of(Lists.transform(tableSchemaPb.getFields(), Field.FROM_PB_FUNCTION)); } }
private String createQuery(TableSchema outputTable) { StringBuilder builder = new StringBuilder(); createTempFunction(builder,outputTable.getFields()); appendInsert(builder,outputTable.getFields()); appendSelect(builder,outputTable.getFields()); doParsing(builder); return builder.toString(); } private void appendSelect(StringBuilder builder, List<TableFieldSchema> fieldsSchema) {
static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { List<TableFieldSchema> fields = tableSchemaPb.getFields(); if (fields == null) { fields = Collections.emptyList(); } return Schema.of(FieldList.fromPb(fields)); } }
private QueryResponse getTypedTableRows(QueryResponse response) { List<TableRow> rows = response.getRows(); TableSchema schema = response.getSchema(); response.setRows( rows.stream() .map(r -> getTypedTableRow(schema.getFields(), r)) .collect(Collectors.toList())); return response; }
/** * Gets the JSON representation of the table's fields. * * @param tableSchema {@link TableSchema} to get JSON fields from * @return the JSON of the fields. * @throws IOException */ static String getTableFieldsJson(TableSchema tableSchema) throws IOException { return JacksonFactory.getDefaultInstance().toString(tableSchema.getFields()); } }
/** * Tries to parse the JSON {@link TableRow} from BigQuery. * * <p>Only supports basic types and arrays. Doesn't support date types. */ public static Row toBeamRow(Schema rowSchema, TableSchema bqSchema, TableRow jsonBqRow) { List<TableFieldSchema> bqFields = bqSchema.getFields(); Map<String, Integer> bqFieldIndices = IntStream.range(0, bqFields.size()) .boxed() .collect(toMap(i -> bqFields.get(i).getName(), i -> i)); List<Object> rawJsonValues = rowSchema .getFields() .stream() .map(field -> bqFieldIndices.get(field.getName())) .map(index -> jsonBqRow.getF().get(index).getV()) .collect(toList()); return IntStream.range(0, rowSchema.getFieldCount()) .boxed() .map(index -> toBeamValue(rowSchema.getField(index).getType(), rawJsonValues.get(index))) .collect(toRow(rowSchema)); }
private static Schema schema(TableSchema schema) { Schema.Builder builder = Schema.newBuilder(); for (TableFieldSchema field : schema.getFields()) { builder.addField(field(field)); } return builder.build(); }
private long writeRows( String tableId, List<TableRow> rows, TableSchema schema, String destinationPattern) throws IOException { Schema avroSchema = BigQueryAvroUtils.toGenericAvroSchema(tableId, schema.getFields()); List<TableRow> rowsToWrite = Lists.newArrayList(); int shard = 0; for (TableRow row : rows) { rowsToWrite.add(row); if (rowsToWrite.size() == 5) { writeRowsHelper(rowsToWrite, avroSchema, destinationPattern, shard++); rowsToWrite.clear(); } } if (!rowsToWrite.isEmpty()) { writeRowsHelper(rowsToWrite, avroSchema, destinationPattern, shard++); } return shard; }
tfs.setFields(subfields); schema.getFields().add(tfs);
@Test public void testToTableSchema_flat() { TableSchema schema = toTableSchema(FLAT_TYPE); assertThat(schema.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
@Test public void testToTableSchema_array() { TableSchema schema = toTableSchema(ARRAY_TYPE); assertThat(schema.getFields(), contains(IDS)); }
@Test public void testToTableSchema_row() { TableSchema schema = toTableSchema(ROW_TYPE); assertThat(schema.getFields().size(), equalTo(1)); TableFieldSchema field = schema.getFields().get(0); assertThat(field.getName(), equalTo("row")); assertThat(field.getType(), equalTo(StandardSQLTypeName.STRUCT.toString())); assertThat(field.getMode(), nullValue()); assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
@Test public void testToTableSchema_array_row() { TableSchema schema = toTableSchema(ARRAY_ROW_TYPE); assertThat(schema.getFields().size(), equalTo(1)); TableFieldSchema field = schema.getFields().get(0); assertThat(field.getName(), equalTo("rows")); assertThat(field.getType(), equalTo(StandardSQLTypeName.STRUCT.toString())); assertThat(field.getMode(), equalTo(Mode.REPEATED.toString())); assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
current = getTypedTableRow(schema.getFields(), iteratorOverCurrentBatch.next()); return true;
? rows : rows.stream() .map(r -> getTypedTableRow(schema.getFields(), r)) .collect(Collectors.toList());
String timestampColumn = importSpec.getSchema().getTimestampColumn(); for (TableFieldSchema tableFieldSchema : schema.getFields()) { String name = tableFieldSchema.getName(); String bigQueryType = tableFieldSchema.getType();
result.setFields(fieldSchema.getFields());