static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
/** @see TableFieldSchema#getType() */ public String getType() { return fieldSchema.getType(); }
/** * Validates that a BigQuery column in Avro format can be used as a valid Datastore Entity key * name. */ public static void validateKeyColumn(TableFieldSchema column, Object columnValue) throws IllegalArgumentException { // Entity key name must be different than null if (columnValue == null) { throw new IllegalArgumentException( String.format( "Column [%s] with NULL value cannot be set as Entity name.", column.getName())); } // Entity key names cannot exceed 1500 bytes, the maximum size of an idex-able Datastore // string property: https://cloud.google.com/datastore/docs/concepts/limits if (column.getType().equals("STRING") && columnValue.toString().getBytes().length > MAX_STRING_SIZE_BYTES) { throw new IllegalArgumentException( String.format( "Column [%s] exceeding %d bytes cannot be set as Entity name.", column.getName(), MAX_STRING_SIZE_BYTES)); } // BigQuery column type must be among the supported ones (ex: cannot be RECORD) if (!SUPPORTED_KEY_NAME_TYPES.contains(column.getType())) { throw new IllegalArgumentException( String.format( "Column [%s] of type %s cannot be set as Entity name.", column.getName(), column.getType())); } }
public void writeField(StringBuilder builder, TableFieldSchema fieldSchema) { if (fieldSchema.getMode().equals("REPEATED") && fieldSchema.getType().equals("RECORD")) { builder.append(fieldSchema.getName()+ " " + "ARRAY<STRUCT<"); writeOutputSchema(builder, fieldSchema.getFields()); builder.append(">>"); } if(fieldSchema.getMode().equals("REPEATED") && !fieldSchema.getType().equals("RECORD") ) { builder.append(fieldSchema.getName()+ " " + "ARRAY<"); builder.append(fieldSchema.getType()); builder.append(">"); } if(!fieldSchema.getMode().equals("REPEATED") && fieldSchema.getType().equals("RECORD") ) { builder.append(fieldSchema.getName()+ " " + "STRUCT<"); writeOutputSchema(builder, fieldSchema.getFields()); builder.append(">"); } if(!fieldSchema.getMode().equals("REPEATED") && !fieldSchema.getType().equals("RECORD") ) { builder.append(fieldSchema.getName() +" "+ fieldSchema.getType()); } }
if (fieldSchema.getType().equals("RECORD")) { @SuppressWarnings("unchecked") Map<String, Object> typedV = (Map<String, Object>) v; if (fieldSchema.getType().equals("FLOAT")) { return Double.parseDouble((String) v); if (fieldSchema.getType().equals("BOOLEAN")) { return Boolean.parseBoolean((String) v); if (fieldSchema.getType().equals("TIMESTAMP")) { return (String) v;
if (fieldSchema.getType().equals("RECORD")) { @SuppressWarnings("unchecked") Map<String, Object> typedV = (Map<String, Object>) v; if (fieldSchema.getType().equals("FLOAT")) { return Double.parseDouble((String) v); if (fieldSchema.getType().equals("BOOLEAN")) { return Boolean.parseBoolean((String) v); if (fieldSchema.getType().equals("TIMESTAMP")) { return BigQueryAvroUtils.formatTimestamp((String) v);
String bqType = fieldSchema.getType(); Type expectedAvroType = BIG_QUERY_TO_AVRO_TYPES.get(bqType); verifyNotNull(expectedAvroType, "Unsupported BigQuery type: %s", bqType); switch (fieldSchema.getType()) { case "STRING": case "DATE": String.format( "Unexpected BigQuery field schema type %s for field named %s", fieldSchema.getType(), fieldSchema.getName()));
private static Field convertField(TableFieldSchema bigQueryField) { Type avroType = BIG_QUERY_TO_AVRO_TYPES.get(bigQueryField.getType()); Schema elementSchema; if (avroType == Type.RECORD) { elementSchema = toGenericAvroSchema(bigQueryField.getName(), bigQueryField.getFields()); } else { elementSchema = Schema.create(avroType); } Schema fieldSchema; if (bigQueryField.getMode() == null || "NULLABLE".equals(bigQueryField.getMode())) { fieldSchema = Schema.createUnion(Schema.create(Type.NULL), elementSchema); } else if ("REQUIRED".equals(bigQueryField.getMode())) { fieldSchema = elementSchema; } else if ("REPEATED".equals(bigQueryField.getMode())) { fieldSchema = Schema.createArray(elementSchema); } else { throw new IllegalArgumentException( String.format("Unknown BigQuery Field Mode: %s", bigQueryField.getMode())); } return new Field( bigQueryField.getName(), fieldSchema, bigQueryField.getDescription(), (Object) null /* Cast to avoid deprecated JsonNode constructor. */); } }
@Test public void testToTableSchema_row() { TableSchema schema = toTableSchema(ROW_TYPE); assertThat(schema.getFields().size(), equalTo(1)); TableFieldSchema field = schema.getFields().get(0); assertThat(field.getName(), equalTo("row")); assertThat(field.getType(), equalTo(StandardSQLTypeName.STRUCT.toString())); assertThat(field.getMode(), nullValue()); assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
@Test public void testToTableSchema_array_row() { TableSchema schema = toTableSchema(ARRAY_ROW_TYPE); assertThat(schema.getFields().size(), equalTo(1)); TableFieldSchema field = schema.getFields().get(0); assertThat(field.getName(), equalTo("rows")); assertThat(field.getType(), equalTo(StandardSQLTypeName.STRUCT.toString())); assertThat(field.getMode(), equalTo(Mode.REPEATED.toString())); assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
private static Type fieldType(TableFieldSchema field) { String type = field.getType(); type = type.toUpperCase(); switch (type) { case "STRING": return Field.Type.string(); case "INT64": return Field.Type.integer(); case "FLOAT64": return Field.Type.floatingPoint(); case "BOOL": return Field.Type.bool(); case "BYTES": return Field.Type.bytes(); case "DATE": return Field.Type.date(); case "DATETIME": return Field.Type.datetime(); case "TIME": return Field.Type.time(); case "TIMESTAMP": return Field.Type.timestamp(); case "RECORD": return Field.Type.record(fieldList(field)); } throw new RuntimeException("Invalid type: " + type); }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.setName(fieldSchemaPb.getName()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.setMode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.setDescription(fieldSchemaPb.getDescription()); } FieldList subFields = fieldSchemaPb.getFields() != null ? FieldList.of(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)) : null; fieldBuilder.setType(LegacySQLTypeName.valueOf(fieldSchemaPb.getType()), subFields); return fieldBuilder.build(); } }
String bigQueryType = tableFieldSchema.getType();
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.name(fieldSchemaPb.getName()); Type.Value enumValue = Type.Value.valueOf(fieldSchemaPb.getType()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.mode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.description(fieldSchemaPb.getDescription()); } if (fieldSchemaPb.getFields() != null) { fieldBuilder.type(Type.record(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION))); } else { fieldBuilder.type(new Type(enumValue)); } return fieldBuilder.build(); } }
static Field fromPb(TableFieldSchema fieldSchemaPb) { Builder fieldBuilder = new Builder(); fieldBuilder.name(fieldSchemaPb.getName()); Type.Value enumValue = Type.Value.valueOf(fieldSchemaPb.getType()); if (fieldSchemaPb.getMode() != null) { fieldBuilder.mode(Mode.valueOf(fieldSchemaPb.getMode())); } if (fieldSchemaPb.getDescription() != null) { fieldBuilder.description(fieldSchemaPb.getDescription()); } if (fieldSchemaPb.getFields() != null) { fieldBuilder.type(Type.record(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION))); } else { fieldBuilder.type(new Type(enumValue)); } return fieldBuilder.build(); } }