public List<Column> getColumns() { return schema.getColumns(); }
public String printRecord(PageReader reader, String delimiter) { StringBuilder sb = new StringBuilder(); for (Column c : schema.getColumns()) { if (c.getIndex() != 0) { sb.append(delimiter); } sb.append(printColumn(reader, c)); } return sb.toString(); }
private Map<String, Integer> newColumnIndex(Schema schema) { ImmutableMap.Builder<String, Integer> builder = ImmutableMap.builder(); for (Column column : schema.getColumns()) { builder.put(column.getName(), column.getIndex()); } return builder.build(); }
public List<String> printRecord(PageReader reader) { for (Column c : schema.getColumns()) { record.set(c.getIndex(), printColumn(reader, c)); } return record; }
private DynamicPageBuilder( final DynamicColumnSetterFactory factory, final BufferAllocator allocator, final Schema schema, final PageOutput output) { this.pageBuilder = new PageBuilder(allocator, schema, output); this.schema = schema; ImmutableList.Builder<DynamicColumnSetter> setters = ImmutableList.builder(); ImmutableMap.Builder<String, DynamicColumnSetter> lookup = ImmutableMap.builder(); for (Column c : schema.getColumns()) { DynamicColumnSetter setter = factory.newColumnSetter(pageBuilder, c); setters.add(setter); lookup.put(c.getName(), setter); } this.setters = setters.build().toArray(new DynamicColumnSetter[0]); this.columnLookup = lookup.build(); }
public static Object[] toObjects(final PageReader record) { final Object[] values = new Object[record.getSchema().getColumns().size()]; record.getSchema().visitColumns(new ObjectColumnVisitor(record) { @Override public void visit(Column column, Object object) { values[column.getIndex()] = object; } }); return values; }
public static TimestampFormatter[] newTimestampColumnFormatters( TimestampFormatter.Task formatterTask, Schema schema, Map<String, ? extends TimestampFormatter.TimestampColumnOption> columnOptions) { TimestampFormatter[] formatters = new TimestampFormatter[schema.getColumnCount()]; int i = 0; for (Column column : schema.getColumns()) { if (column.getType() instanceof TimestampType) { Optional<TimestampFormatter.TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName())); formatters[i] = TimestampFormatter.of(formatterTask, option); } i++; } return formatters; } }
@Override public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) { final PluginTask task = config.loadConfig(PluginTask.class); if (task.getPrintsColumnNames()) { for (final Column column : schema.getColumns()) { if (column.getIndex() > 0) { System.out.print(","); } System.out.print(column.getName()); } System.out.println(""); } return resume(task.dump(), schema, taskCount, control); }
private Schema applyLowerToUpperRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toUpperCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
private Schema applyUpperToLowerRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toLowerCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
for (Column column : inputSchema.getColumns()) { String name = column.getName(); if (name.matches(regexBuilder.toString())) {
private static Row newRow(Schema schema) { ColumnValue[] values = new ColumnValue[schema.getColumnCount()]; for (Column column : schema.getColumns()) { values[column.getIndex()] = newValue(column); } return new Row(values); }
for (Column column : inputSchema.getColumns()) { schemaBuilder.add(column.getName().replaceAll(regexBuilder.toString(), replace), column.getType());
private void writeHeader(Schema schema, LineEncoder encoder, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString) { String delimiterString = String.valueOf(delimiter); for (Column column : schema.getColumns()) { if (column.getIndex() != 0) { encoder.addText(delimiterString); } encoder.addText(setEscapeAndQuoteValue(column.getName(), delimiter, policy, quote, escape, newline, nullString)); } encoder.addNewLine(); }
private Schema applyTruncateRule(Schema inputSchema, TruncateRule rule) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { if (column.getName().length() <= rule.getMaxLength()) { builder.add(column.getName(), column.getType()); } else { try { builder.add(column.getName().substring(0, rule.getMaxLength()), column.getType()); } catch (IndexOutOfBoundsException ex) { logger.error("FATAL unexpected error in \"truncate\" rule: substring failed."); throw new AssertionError("FATAL unexpected error in \"truncate\" rule: substring failed.", ex); } } } return builder.build(); }
private Schema applyRegexReplaceRule(Schema inputSchema, RegexReplaceRule rule) { final String match = rule.getMatch(); final String replace = rule.getReplace(); Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { // TODO(dmikurube): Check if we need a kind of sanitization? try { builder.add(column.getName().replaceAll(match, replace), column.getType()); } catch (PatternSyntaxException ex) { throw new ConfigException(ex); } } return builder.build(); }
for (Column column : inputSchema.getColumns()) { if (!removeColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType())); for (Column column : inputSchema.getColumns()) { if (keepColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType()));
@Override public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control) { PluginTask task = config.loadConfig(PluginTask.class); Map<String, String> renameMap = task.getRenameMap(); List<ConfigSource> rulesList = task.getRulesList(); // Check if the given column in "columns" exists or not. for (String columnName : renameMap.keySet()) { inputSchema.lookupColumn(columnName); // throws SchemaConfigException } // Rename by "columns": to be applied before "rules". Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { String name = column.getName(); if (renameMap.containsKey(name)) { name = renameMap.get(name); } builder.add(name, column.getType()); } Schema intermediateSchema = builder.build(); // Rename by "rules". Schema outputSchema = intermediateSchema; for (ConfigSource rule : rulesList) { outputSchema = applyRule(rule, intermediateSchema); intermediateSchema = outputSchema; } control.run(task.dump(), outputSchema); }
private static List<List<String>> parse(CsvParserPlugin.PluginTask task, FileInput input) { LineDecoder decoder = new LineDecoder(input, task); CsvTokenizer tokenizer = new CsvTokenizer(decoder, task); Schema schema = task.getSchemaConfig().toSchema(); tokenizer.nextFile(); List<List<String>> records = new ArrayList<>(); while (tokenizer.nextRecord()) { List<String> record = new ArrayList<>(); for (Column c : schema.getColumns()) { String v = tokenizer.nextColumnOrNull(); record.add(v); } records.add(record); } return records; }
@Override public void run(TaskSource task, Schema newSchema) { ArrayList<String> resolvedColumnNamesList = new ArrayList<>(newSchema.size()); for (Column resolvedColumn : newSchema.getColumns()) { resolvedColumnNamesList.add(resolvedColumn.getName()); } String[] resolvedColumnNames = Iterables.toArray(resolvedColumnNamesList, String.class); assertEquals(expectedColumnNames, resolvedColumnNames); for (int i = 0; i < expectedColumnNames.length; ++i) { Column original = originalSchema.getColumn(i); Column resolved = newSchema.getColumn(i); assertEquals(original.getType(), resolved.getType()); } } });