/** * Creates a {@link RowTypeInfo} with projected fields. * * @param rowType The original RowTypeInfo whose fields are projected * @param fieldMapping The field mapping of the projection * @return A RowTypeInfo with projected fields. */ public static RowTypeInfo projectFields(RowTypeInfo rowType, int[] fieldMapping) { TypeInformation[] fieldTypes = new TypeInformation[fieldMapping.length]; String[] fieldNames = new String[fieldMapping.length]; for (int i = 0; i < fieldMapping.length; i++) { fieldTypes[i] = rowType.getTypeAt(fieldMapping[i]); fieldNames[i] = rowType.getFieldNames()[fieldMapping[i]]; } return new RowTypeInfo(fieldTypes, fieldNames); } }
private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { if (reuse == null) { reuse = mapper.createObjectNode(); } final String[] fieldNames = info.getFieldNames(); final TypeInformation<?>[] fieldTypes = info.getFieldTypes(); // validate the row if (row.getArity() != fieldNames.length) { throw new IllegalStateException(String.format( "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); } for (int i = 0; i < fieldNames.length; i++) { final String name = fieldNames[i]; final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); reuse.set(name, fieldConverted); } return reuse; }
private Row convertRow(JsonNode node, RowTypeInfo info) { final String[] names = info.getFieldNames(); final TypeInformation<?>[] types = info.getFieldTypes(); final Row row = new Row(names.length); for (int i = 0; i < names.length; i++) { final String name = names[i]; final JsonNode subNode = node.get(name); if (subNode == null) { if (failOnMissingField) { throw new IllegalStateException( "Could not find field with name '" + name + "'."); } else { row.setField(i, null); } } else { row.setField(i, convert(subNode, types[i])); } } return row; }
} else if (typeInfo instanceof RowTypeInfo) { final RowTypeInfo rt = (RowTypeInfo) typeInfo; final String[] fieldNames = rt.getFieldNames(); final TypeInformation<?>[] fieldTypes = rt.getFieldTypes();
@Test public void testCustomFieldNames() { String[] fieldNames = new String[]{"int", "row", "string"}; RowTypeInfo typeInfo1 = new RowTypeInfo(typeList, new String[]{"int", "row", "string"}); assertArrayEquals(new String[]{"int", "row", "string"}, typeInfo1.getFieldNames()); assertEquals(BasicTypeInfo.STRING_TYPE_INFO, typeInfo1.getTypeAt("string")); assertEquals(BasicTypeInfo.STRING_TYPE_INFO, typeInfo1.getTypeAt(2)); assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, typeInfo1.getTypeAt("row.0")); assertEquals(BasicTypeInfo.BIG_DEC_TYPE_INFO, typeInfo1.getTypeAt("row.f1")); // change the names in fieldNames fieldNames[1] = "composite"; RowTypeInfo typeInfo2 = new RowTypeInfo(typeList, fieldNames); // make sure the field names are deep copied assertArrayEquals(new String[]{"int", "row", "string"}, typeInfo1.getFieldNames()); assertArrayEquals(new String[]{"int", "composite", "string"}, typeInfo2.getFieldNames()); }
private OrcTableSource(String path, TypeDescription orcSchema, Configuration orcConfig, int batchSize, boolean recursiveEnumeration, int[] selectedFields, Predicate[] predicates) { Preconditions.checkNotNull(path, "Path must not be null."); Preconditions.checkNotNull(orcSchema, "OrcSchema must not be null."); Preconditions.checkNotNull(path, "Configuration must not be null."); Preconditions.checkArgument(batchSize > 0, "Batch size must be larger than null."); this.path = path; this.orcSchema = orcSchema; this.orcConfig = orcConfig; this.batchSize = batchSize; this.recursiveEnumeration = recursiveEnumeration; this.selectedFields = selectedFields; this.predicates = predicates; // determine the type information from the ORC schema RowTypeInfo typeInfoFromSchema = (RowTypeInfo) OrcBatchReader.schemaToTypeInfo(this.orcSchema); // set return type info if (selectedFields == null) { this.typeInfo = typeInfoFromSchema; } else { this.typeInfo = RowTypeInfo.projectFields(typeInfoFromSchema, selectedFields); } // create a TableSchema that corresponds to the ORC schema this.tableSchema = new TableSchema( typeInfoFromSchema.getFieldNames(), typeInfoFromSchema.getFieldTypes() ); }
@Test public void testGetTypeAt() { RowTypeInfo typeInfo = new RowTypeInfo(typeList); assertArrayEquals(new String[]{"f0", "f1", "f2"}, typeInfo.getFieldNames()); assertEquals(BasicTypeInfo.STRING_TYPE_INFO, typeInfo.getTypeAt("f2")); assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, typeInfo.getTypeAt("f1.f0")); assertEquals(BasicTypeInfo.BIG_DEC_TYPE_INFO, typeInfo.getTypeAt("f1.1")); }
@Override public String[] getFieldNames() { return type.getFieldNames(); }
MockTableSource(List<Row> data, RowTypeInfo type) { this.data = data; this.type = type; this.schema = new TableSchema(type.getFieldNames(), type.getFieldTypes()); }
ExternalCatalogTable toExternalCatalogTable() { TableSchema tableSchema = new TableSchema(schema.getFieldNames(), schema.getFieldTypes()); ConnectorDescriptor descriptor = new ConnectorDescriptor(CONNECTOR_TYPE, CONNECTOR_VERSION, false) { @Override public void addConnectorProperties(DescriptorProperties properties) { properties.putTableSchema(TABLE_SCHEMA_CONNECTOR_PROPERTY, tableSchema); properties.putString(TABLE_DATA_CONNECTOR_PROPERTY, serializeRows()); } }; return new ExternalCatalogTable(descriptor, Option.empty(), Option.empty(), Option.empty(), Option.empty()); }
/** * Creates a JSON deserialization schema for the given fields and types. * * @param typeInfo Type information describing the result type. The field names are used * to parse the JSON file and so are the types. */ public JsonRowDeserializationSchema(TypeInformation<Row> typeInfo) { Preconditions.checkNotNull(typeInfo, "Type information"); this.typeInfo = typeInfo; this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); }
/** * Creates a JSON deserialization schema for the given fields and types. * * @param typeInfo Type information describing the result type. The field names are used * to parse the JSON file and so are the types. */ public JsonRowDeserializationSchema(TypeInformation<Row> typeInfo) { Preconditions.checkNotNull(typeInfo, "Type information"); this.typeInfo = typeInfo; this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); }
@Override public TypeInformation getProducedType() { return new BaseRowTypeInfo(rowTypeInfo.getFieldTypes(), rowTypeInfo.getFieldNames()); }
/** * Creates a {@link RowTypeInfo} with projected fields. * * @param rowType The original RowTypeInfo whose fields are projected * @param fieldMapping The field mapping of the projection * @return A RowTypeInfo with projected fields. */ public static RowTypeInfo projectFields(RowTypeInfo rowType, int[] fieldMapping) { TypeInformation[] fieldTypes = new TypeInformation[fieldMapping.length]; String[] fieldNames = new String[fieldMapping.length]; for (int i = 0; i < fieldMapping.length; i++) { fieldTypes[i] = rowType.getTypeAt(fieldMapping[i]); fieldNames[i] = rowType.getFieldNames()[fieldMapping[i]]; } return new RowTypeInfo(fieldTypes, fieldNames); } }
@Override public Row deserialize(byte[] messageKey, byte[] message, String topic, int partition, long offset) throws IOException { @SuppressWarnings("unchecked") Map<String, Object> map = MAPPER.readValue(message, Map.class); String[] names = rowTypeInfo.getFieldNames(); Row row = new Row(names.length); for (int i = 0; i < names.length; i++) { row.setField(i, map.get(names[i])); } return row; }
/** * Creates a {@link RowTypeInfo} with projected fields. * * @param rowType The original RowTypeInfo whose fields are projected * @param fieldMapping The field mapping of the projection * @return A RowTypeInfo with projected fields. */ public static RowTypeInfo projectFields(RowTypeInfo rowType, int[] fieldMapping) { TypeInformation[] fieldTypes = new TypeInformation[fieldMapping.length]; String[] fieldNames = new String[fieldMapping.length]; for (int i = 0; i < fieldMapping.length; i++) { fieldTypes[i] = rowType.getTypeAt(fieldMapping[i]); fieldNames[i] = rowType.getFieldNames()[fieldMapping[i]]; } return new RowTypeInfo(fieldTypes, fieldNames); } }
@SuppressWarnings("unchecked") @Override public DataType getReturnType() { return TypeConverters.createInternalTypeFromTypeInfo(new BaseRowTypeInfo( rowTypeInfo.getFieldTypes(), rowTypeInfo.getFieldNames())); }
/** * Creates a JSON serialization schema for the given fields and types. * * @param rowSchema The schema of the rows to encode. */ public JsonRowSerializationSchema(RowTypeInfo rowSchema) { Preconditions.checkNotNull(rowSchema); String[] fieldNames = rowSchema.getFieldNames(); TypeInformation[] fieldTypes = rowSchema.getFieldTypes(); // check that no field is composite for (int i = 0; i < fieldTypes.length; i++) { if (fieldTypes[i] instanceof CompositeType) { throw new IllegalArgumentException("JsonRowSerializationSchema cannot encode rows with nested schema, " + "but field '" + fieldNames[i] + "' is nested: " + fieldTypes[i].toString()); } } this.fieldNames = fieldNames; }
private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { if (reuse == null) { reuse = mapper.createObjectNode(); } final String[] fieldNames = info.getFieldNames(); final TypeInformation<?>[] fieldTypes = info.getFieldTypes(); // validate the row if (row.getArity() != fieldNames.length) { throw new IllegalStateException(String.format( "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); } for (int i = 0; i < fieldNames.length; i++) { final String name = fieldNames[i]; final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); reuse.set(name, fieldConverted); } return reuse; }
private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { if (reuse == null) { reuse = mapper.createObjectNode(); } final String[] fieldNames = info.getFieldNames(); final TypeInformation<?>[] fieldTypes = info.getFieldTypes(); // validate the row if (row.getArity() != fieldNames.length) { throw new IllegalStateException(String.format( "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); } for (int i = 0; i < fieldNames.length; i++) { final String name = fieldNames[i]; final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); reuse.set(name, fieldConverted); } return reuse; }