/** * Set single index. */ public void addSingleIndex(Index index) { Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); addIndex(index, columnNames); }
/** * Set the unique keys Maybe one or more keys are combined to be a unique key. */ public void setUniqueKeys(List<List<String>> uniqueKeys) { checkArgument(this.uniqueKeys.size() == 0, "unique key has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (List<String> uk : uniqueKeys) { for (String key : uk) { if (!columnNames.contains(key)) { throw new IllegalArgumentException("The unique key '" + key + "' is not in the table schema"); } } this.uniqueKeys.add(uk); } }
/** * Set the indexes. */ public void setIndexes(List<Index> keys) { checkArgument(indexes.size() == 0, "indexes has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (Index index : keys) { addIndex(index, columnNames); } }
/** * Set the primary keys Maybe one or more keys are combined to be a primary key. * * @param keys one or more combined primary key */ public void setPrimaryKey(String... keys) { checkArgument(primaryKeys.size() == 0, "primary key has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (String key : keys) { if (!columnNames.contains(key)) { throw new IllegalArgumentException("The primary key '" + key + "' is not in the table schema"); } primaryKeys.add(key); } }
/** * Set the header fields which is in the properties of source record. * * @param fields header field names */ public void setHeaderFields(List<String> fields) { checkArgument(headerFields.size() == 0, "header fields has been set"); Set<String> columnNames = new HashSet<>(); columnNames.addAll(Arrays.asList(getColumnNames())); for (String field : fields) { if (!columnNames.contains(field)) { throw new IllegalArgumentException("The HEADER column field '" + field + "' is not in the table schema"); } headerFields.add(field); } }
public IndexKey toIndexKey(RichTableSchema schema) { assert null != schema; String[] fieldNames = schema.getColumnNames(); int[] indexes = new int[keyList.size()]; for (int i = 0; i < keyList.size(); i++) { int idx = findIndex(keyList.get(i), fieldNames); indexes[i] = idx; } return IndexKey.of(unique, indexes); }
/** * Returns the final result type info of this table. * This type info including the computed columns (if exist) and exclude proctime field. */ public RowType getResultType() { return new RowType(getColumnTypes(), getColumnNames()); }
/** * Returns the final result type info of this table. * This type info including the computed columns (if exist) and exclude proctime field. */ public DataType getResultRowType() { return DataTypes.createRowType(getColumnTypes(), getColumnNames()); }
private OrcVectorizedColumnRowTableSource createSource(Map<String, String> props) { TableProperties properties = new TableProperties(); properties.putProperties(props); RichTableSchema schema = properties.readSchemaFromProperties(null); String filePath = properties.getString(ORCOptions.FILE_PATH); if (StringUtils.isNullOrWhitespaceOnly(filePath)) { throw new RuntimeException(ORCOptions.PARAMS_HELP_MSG); } boolean enumerateNestedFiles = properties.getBoolean(ORCOptions.ENUMERATE_NESTED_FILES); InternalType[] dataTypes = Arrays.stream(schema.getColumnTypes()).toArray(InternalType[]::new); OrcVectorizedColumnRowTableSource t = new OrcVectorizedColumnRowTableSource( new Path(filePath), dataTypes, schema.getColumnNames(), enumerateNestedFiles); t.setSchemaFields(schema.getColumnNames()); return t; }
private ParquetVectorizedColumnRowTableSource getSource(Map<String, String> props) { TableProperties tableProperties = new TableProperties(); tableProperties.putProperties(props); RichTableSchema richTableSchema = tableProperties.readSchemaFromProperties(null); String filePath = tableProperties.getString(ParquetOptions.FILE_PATH); if (StringUtils.isNullOrWhitespaceOnly(filePath)) { throw new RuntimeException(ParquetOptions.PARAMS_HELP_MSG); } boolean enumerateNestedFiles = tableProperties.getBoolean(ParquetOptions.ENUMERATE_NESTED_FILES); return new ParquetVectorizedColumnRowTableSource( new Path(filePath), richTableSchema.getColumnTypes(), richTableSchema.getColumnNames(), enumerateNestedFiles); }
.lineDelimiter(lineDelim) .charset(charset) .fields(schema.getColumnNames(), schema.getColumnTypes(), schema.getNullables()) .timezone(tz) .setNestedFileEnumerate(enumerateNestedFiles);
.configure(schema.getColumnNames(), schema.getColumnTypes());
.configure(schema.getColumnNames(), schema.getColumnTypes());
.configure(schema.getColumnNames(), schema.getColumnTypes());
Option.empty(), Option.apply(tz) ).configure(schema.getColumnNames(), schema.getColumnTypes()); case "retract": return Option.empty(), Option.apply(tz) ).configure(schema.getColumnNames(), schema.getColumnTypes()); case "upsert": return new UpsertCsvTableSink( Option.empty(), Option.apply(tz) ).configure(schema.getColumnNames(), schema.getColumnTypes()); default: throw new RuntimeException("Unsupported updateMode: " + updateMode + " for CSV sink.");