/** * Returns the final result type info of this table. * This type info including the computed columns (if exist) and exclude proctime field. */ public RowType getResultType() { return new RowType(getColumnTypes(), getColumnNames()); }
public RowTypeInfo getResultTypeInfo() { return (RowTypeInfo) TypeConverters.createExternalTypeInfoFromDataType(getResultRowType()); }
/** * Set single index. */ public void addSingleIndex(Index index) { Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); addIndex(index, columnNames); }
tableBaseInfo.setHeaderFields(schema.getHeaderFields()) .setRowTypeInfo(toRowTypeInfo(schema.getResultRowType())) .setPrimaryKeys(schema.getPrimaryKeys()) .setUserParamsMap(properties.toMap()); .setProperties(prop) .setTopic(topic) .setRowTypeInfo(toRowTypeInfo(schema.getResultRowType())); .configure(schema.getColumnNames(), schema.getColumnTypes());
.lineDelimiter(lineDelim) .charset(charset) .fields(schema.getColumnNames(), schema.getColumnTypes(), schema.getNullables()) .timezone(tz) .setNestedFileEnumerate(enumerateNestedFiles); if (!schema.getPrimaryKeys().isEmpty()) { uniqueKeys.add(new HashSet<>(schema.getPrimaryKeys())); for (List<String> uniqueKey : schema.getUniqueKeys()) { uniqueKeys.add(new HashSet<>(uniqueKey)); for (RichTableSchema.Index index : schema.getIndexes()) { if (index.unique) { uniqueKeys.add(new HashSet<>(index.keyList));
/** * Set the unique keys Maybe one or more keys are combined to be a unique key. */ public void setUniqueKeys(List<List<String>> uniqueKeys) { checkArgument(this.uniqueKeys.size() == 0, "unique key has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (List<String> uk : uniqueKeys) { for (String key : uk) { if (!columnNames.contains(key)) { throw new IllegalArgumentException("The unique key '" + key + "' is not in the table schema"); } } this.uniqueKeys.add(uk); } }
private Kafka09TableSource createSource(Map<String, String> props) { TableProperties properties = new TableProperties(); properties.putProperties(props); RichTableSchema schema = properties.readSchemaFromProperties(null); String topicStr = properties.getString(KafkaOptions.TOPIC); String topicPatternStr = properties.getString(KafkaOptions.TOPIC_PATTERN); Properties prop = getProperties( Kafka09Options.ESSENTIAL_CONSUMER_KEYS, Kafka09Options.OPTIONAL_CONSUMER_KEYS, properties); // TODO: support batch mode. boolean isBatchMode = false; if (!StringUtils.isNullOrWhitespaceOnly(topicStr)) { List<String> topics = Arrays.asList(topicStr.split(",")); return new Kafka09TableSource(topics, null, prop, getStartupMode(properties), -1, isBatchMode, TypeConverters.toBaseRowTypeInfo(schema.getResultType())); } else if (!StringUtils.isNullOrWhitespaceOnly(topicPatternStr)) { return new Kafka09TableSource(null, topicPatternStr, prop, getStartupMode(properties), -1, isBatchMode, TypeConverters.toBaseRowTypeInfo(schema.getResultType())); } else { throw new RuntimeException("No sufficient parameters for Kafka09." + "topic or topic pattern needed."); } }
public CatalogTable( String tableType, TableSchema tableSchema, TableStats tableStats, Map<String, String> properties, boolean isStreaming) { this.tableType = tableType; this.tableSchema = tableSchema; this.tableStats = tableStats; this.properties = properties; this.isStreaming = isStreaming; this.richTableSchema = new RichTableSchema(tableSchema.getFieldNames(), tableSchema.getFieldTypes()); }
tableBaseInfo.setHeaderFields(schema.getHeaderFields()) .setRowTypeInfo(toRowTypeInfo(schema.getResultRowType())) .setPrimaryKeys(schema.getPrimaryKeys()) .setUserParamsMap(properties.toMap()); .setProperties(prop) .setTopic(topic) .setRowTypeInfo(toRowTypeInfo(schema.getResultRowType())); .configure(schema.getColumnNames(), schema.getColumnTypes());
/** * Set the primary keys Maybe one or more keys are combined to be a primary key. * * @param keys one or more combined primary key */ public void setPrimaryKey(String... keys) { checkArgument(primaryKeys.size() == 0, "primary key has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (String key : keys) { if (!columnNames.contains(key)) { throw new IllegalArgumentException("The primary key '" + key + "' is not in the table schema"); } primaryKeys.add(key); } }
/** * Set the indexes. */ public void setIndexes(List<Index> keys) { checkArgument(indexes.size() == 0, "indexes has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (Index index : keys) { addIndex(index, columnNames); } }
List<String> topics = Arrays.asList(topicStr.split(",")); return new Kafka010TableSource(topics, null, prop, getStartupMode(properties), startInMs, isBatchMode, TypeConverters.toBaseRowTypeInfo(schema.getResultType())); } else if (!StringUtils.isNullOrWhitespaceOnly(topicPatternStr)) { return new Kafka010TableSource(null, topicPatternStr, prop, getStartupMode(properties), startInMs, isBatchMode, TypeConverters.toBaseRowTypeInfo(schema.getResultType())); } else { throw new RuntimeException("No sufficient parameters for Kafka010." +
/** * Create an CatalogTable from Hive table. */ public static CatalogTable createCatalogTable(Table hiveTable, TableSchema tableSchema, TableStats tableStats) { return new CatalogTable( "hive", tableSchema, getPropertiesFromHiveTable(hiveTable), new RichTableSchema(tableSchema.getFieldNames(), tableSchema.getFieldTypes()), tableStats, null, getPartitionCols(hiveTable), hiveTable.getPartitionKeysSize() != 0, null, null, -1L, (long) hiveTable.getCreateTime(), (long) hiveTable.getLastAccessTime(), false); }
tableBaseInfo.setHeaderFields(schema.getHeaderFields()) .setRowTypeInfo(SourceUtils.toRowTypeInfo(schema.getResultRowType())) .setPrimaryKeys(schema.getPrimaryKeys()) .setUserParamsMap(properties.toMap()); .setProperties(prop) .setTopic(topic) .setRowTypeInfo(SourceUtils.toRowTypeInfo(schema.getResultRowType())); .configure(schema.getColumnNames(), schema.getColumnTypes());
/** * Returns the final result type info of this table. * This type info including the computed columns (if exist) and exclude proctime field. */ public DataType getResultRowType() { return DataTypes.createRowType(getColumnTypes(), getColumnNames()); }
/** * Set the header fields which is in the properties of source record. * * @param fields header field names */ public void setHeaderFields(List<String> fields) { checkArgument(headerFields.size() == 0, "header fields has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (String field : fields) { if (!columnNames.contains(field)) { throw new IllegalArgumentException("The HEADER column field '" + field + "' is not in the table schema"); } headerFields.add(field); } }
List<String> topics = Arrays.asList(topicStr.split(",")); return new Kafka011TableSource(topics, null, prop, getStartupMode(properties), startInMs, isBatchMode, TypeConverters.toBaseRowTypeInfo(schema.getResultType())); } else if (!StringUtils.isNullOrWhitespaceOnly(topicPatternStr)) { return new Kafka011TableSource(null, topicPatternStr, prop, getStartupMode(properties), startInMs, isBatchMode, TypeConverters.toBaseRowTypeInfo(schema.getResultType())); } else { throw new RuntimeException("No sufficient parameters for Kafka011, " +
private OrcVectorizedColumnRowTableSource createSource(Map<String, String> props) { TableProperties properties = new TableProperties(); properties.putProperties(props); RichTableSchema schema = properties.readSchemaFromProperties(null); String filePath = properties.getString(ORCOptions.FILE_PATH); if (StringUtils.isNullOrWhitespaceOnly(filePath)) { throw new RuntimeException(ORCOptions.PARAMS_HELP_MSG); } boolean enumerateNestedFiles = properties.getBoolean(ORCOptions.ENUMERATE_NESTED_FILES); InternalType[] dataTypes = Arrays.stream(schema.getColumnTypes()).toArray(InternalType[]::new); OrcVectorizedColumnRowTableSource t = new OrcVectorizedColumnRowTableSource( new Path(filePath), dataTypes, schema.getColumnNames(), enumerateNestedFiles); t.setSchemaFields(schema.getColumnNames()); return t; }
public IndexKey toIndexKey(RichTableSchema schema) { assert null != schema; String[] fieldNames = schema.getColumnNames(); int[] indexes = new int[keyList.size()]; for (int i = 0; i < keyList.size(); i++) { int idx = findIndex(keyList.get(i), fieldNames); indexes[i] = idx; } return IndexKey.of(unique, indexes); }
private ParquetVectorizedColumnRowTableSource getSource(Map<String, String> props) { TableProperties tableProperties = new TableProperties(); tableProperties.putProperties(props); RichTableSchema richTableSchema = tableProperties.readSchemaFromProperties(null); String filePath = tableProperties.getString(ParquetOptions.FILE_PATH); if (StringUtils.isNullOrWhitespaceOnly(filePath)) { throw new RuntimeException(ParquetOptions.PARAMS_HELP_MSG); } boolean enumerateNestedFiles = tableProperties.getBoolean(ParquetOptions.ENUMERATE_NESTED_FILES); return new ParquetVectorizedColumnRowTableSource( new Path(filePath), richTableSchema.getColumnTypes(), richTableSchema.getColumnNames(), enumerateNestedFiles); }