private Schema applyLowerToUpperRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toUpperCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
public Schema build() { return new Schema(columns.build()); } }
public static Object[] toObjects(final PageReader record) { final Object[] values = new Object[record.getSchema().getColumns().size()]; record.getSchema().visitColumns(new ObjectColumnVisitor(record) { @Override public void visit(Column column, Object object) { values[column.getIndex()] = object; } }); return values; }
public TablePreviewPrinter(PrintStream out, ModelManager modelManager, Schema schema) { super(out, modelManager, schema); this.samples = new ArrayList<String[]>(); String[] header = new String[schema.getColumnCount()]; for (int i = 0; i < header.length; i++) { header[i] = schema.getColumnName(i) + ":" + schema.getColumnType(i); } samples.add(header); }
private static int maxColumnNameLength(Schema schema) { int max = 0; for (int i = 0; i < schema.getColumnCount(); i++) { max = Math.max(max, schema.getColumnName(i).length()); } return max; }
private static int maxColumnTypeNameLength(Schema schema) { int max = 0; for (int i = 0; i < schema.getColumnCount(); i++) { max = Math.max(max, schema.getColumnType(i).toString().length()); } return max; }
@VisibleForTesting Schema newSchema() { return Schema.builder().add("record", Types.JSON).build(); // generate a schema }
@Test public void buildOutputSchema_AddColumns() { PluginTask task = taskFromYamlString( "type: column", "add_columns:", " - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}", " - {name: string, type: string, default: string}", " - {name: boolean, type: boolean, default: true}", " - {name: long, type: long, default: 0}", " - {name: double, type: double, default: 0.5}", " - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }"); Schema inputSchema = Schema.builder() .add("keep_me", STRING) .build(); Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema); assertEquals(7, outputSchema.size()); Column column; { column = outputSchema.getColumn(0); assertEquals("keep_me", column.getName()); } { column = outputSchema.getColumn(1); assertEquals("timestamp", column.getName()); } }
public List<Column> getColumns() { return schema.getColumns(); }
throw new ConfigException("\"max_length\" in rule \"unique_number_suffix\" must be larger than \"digits\""); int digitsOfNumberOfColumns = Integer.toString(inputSchema.getColumnCount() + offset - 1).length(); if (maxLength.isPresent() && maxLength.get() <= digitsOfNumberOfColumns) { throw new ConfigException("\"max_length\" in rule \"unique_number_suffix\" must be larger than digits of ((number of columns) + \"offset\" - 1)"); for (Column column : inputSchema.getColumns()) { originalColumnNames.add(column.getName()); columnNameCountups.put(column.getName(), offset); Schema.Builder outputBuilder = Schema.builder(); for (Column column : inputSchema.getColumns()) { String truncatedName = column.getName(); if (column.getName().length() > maxLength.or(Integer.MAX_VALUE)) {
int[] indexMapping = new int[inputSchema.size()]; for (int i = 0; i < indexMapping.length; i++) { indexMapping[i] = -1; for (Column column : inputSchema.getColumns()) { if (!removeColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType())); for (Column column : inputSchema.getColumns()) { if (keepColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType())); control.run(task.dump(), new Schema(outputColumns.build()));
@Override public void run(TaskSource task, Schema newSchema) { ArrayList<String> resolvedColumnNamesList = new ArrayList<>(newSchema.size()); for (Column resolvedColumn : newSchema.getColumns()) { resolvedColumnNamesList.add(resolvedColumn.getName()); } String[] resolvedColumnNames = Iterables.toArray(resolvedColumnNamesList, String.class); assertEquals(expectedColumnNames, resolvedColumnNames); for (int i = 0; i < expectedColumnNames.length; ++i) { Column original = originalSchema.getColumn(i); Column resolved = newSchema.getColumn(i); assertEquals(original.getType(), resolved.getType()); } } });
@Override public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control) { PluginTask task = config.loadConfig(PluginTask.class); Map<String, String> renameMap = task.getRenameMap(); List<ConfigSource> rulesList = task.getRulesList(); // Check if the given column in "columns" exists or not. for (String columnName : renameMap.keySet()) { inputSchema.lookupColumn(columnName); // throws SchemaConfigException } // Rename by "columns": to be applied before "rules". Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { String name = column.getName(); if (renameMap.containsKey(name)) { name = renameMap.get(name); } builder.add(name, column.getType()); } Schema intermediateSchema = builder.build(); // Rename by "rules". Schema outputSchema = intermediateSchema; for (ConfigSource rule : rulesList) { outputSchema = applyRule(rule, intermediateSchema); intermediateSchema = outputSchema; } control.run(task.dump(), outputSchema); }
private static Row newRow(Schema schema) { ColumnValue[] values = new ColumnValue[schema.getColumnCount()]; for (Column column : schema.getColumns()) { values[column.getIndex()] = newValue(column); } return new Row(values); }
public void add(Page page) { pageReader.setPage(page); while (pageReader.nextRecord()) { schema.visitColumns(new ColumnVisitor() { public void booleanColumn(Column column) { addDelimiter(column);
int i = 0; if (dropColumns.size() > 0) { for (Column inputColumn : inputSchema.getColumns()) { String name = inputColumn.getName(); boolean matched = false; Column inputColumn; try { inputColumn = inputSchema.lookupColumn(srcName); for (Column column : inputSchema.getColumns()) { Column outputColumn = new Column(i++, column.getName(), column.getType()); builder.add(outputColumn); srcName = src.get(); try { inputColumn = inputSchema.lookupColumn(srcName); return new Schema(builder.build());
public PreviewPrinter(PrintStream out, ModelManager modelManager, Schema schema) { this.out = out; this.modelManager = modelManager; this.schema = schema; this.stringValues = new String[schema.getColumnCount()]; }
static int[] columnOffsets(Schema schema) { int[] offsets = new int[schema.getColumnCount()]; if (!schema.isEmpty()) { offsets[0] = recordHeaderSize(schema); for (int i = 0; i < schema.getColumnCount() - 1; i++) { offsets[i + 1] = offsets[i] + schema.getColumnType(i).getFixedStorageSize(); } } return offsets; } }
public Type getColumnType(int index) { return getColumn(index).getType(); }
private void buildOutputInputColumnMap() { for (Column outputColumn : outputSchema.getColumns()) { String name = outputColumn.getName(); String srcName = getSrc(name, task.getColumns()); if (srcName == null) { srcName = getSrc(name, task.getAddColumns()); } if (srcName == null) { srcName = name; } Column inputColumn; try { inputColumn = inputSchema.lookupColumn(srcName); } catch (SchemaConfigException ex) { inputColumn = null; } outputInputColumnMap.put(outputColumn, inputColumn); // NOTE: inputColumn would be null } }