public Column lookupColumn(String name) { for (Column c : columns) { if (c.getName().equals(name)) { return c; } } throw new SchemaConfigException(String.format("Column '%s' is not found", name)); }
public String getColumnName(int index) { return getColumn(index).getName(); }
@Override public String toString() { StringBuilder sbuf = new StringBuilder(); sbuf.append("Schema{\n"); for (Column c : columns) { sbuf.append(String.format(" %4d: %s %s%n", c.getIndex(), c.getName(), c.getType())); } sbuf.append("}"); return sbuf.toString(); } }
private Map<String, Integer> newColumnIndex(Schema schema) { ImmutableMap.Builder<String, Integer> builder = ImmutableMap.builder(); for (Column column : schema.getColumns()) { builder.put(column.getName(), column.getIndex()); } return builder.build(); }
private DynamicPageBuilder.ColumnOption getColumnOption(Column column) { ConfigSource option = task.getColumnOptions().get(column.getName()); if (option != null) { return option.loadConfig(DynamicPageBuilder.ColumnOption.class); } else { return null; } }
private DynamicPageBuilder( final DynamicColumnSetterFactory factory, final BufferAllocator allocator, final Schema schema, final PageOutput output) { this.pageBuilder = new PageBuilder(allocator, schema, output); this.schema = schema; ImmutableList.Builder<DynamicColumnSetter> setters = ImmutableList.builder(); ImmutableMap.Builder<String, DynamicColumnSetter> lookup = ImmutableMap.builder(); for (Column c : schema.getColumns()) { DynamicColumnSetter setter = factory.newColumnSetter(pageBuilder, c); setters.add(setter); lookup.put(c.getName(), setter); } this.setters = setters.build().toArray(new DynamicColumnSetter[0]); this.columnLookup = lookup.build(); }
@Override public String toString() { return String.format("Column{index:%d, name:%s, type:%s}", getIndex(), getName(), getType().getName()); } }
private Schema applyTruncateRule(Schema inputSchema, TruncateRule rule) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { if (column.getName().length() <= rule.getMaxLength()) { builder.add(column.getName(), column.getType()); } else { try { builder.add(column.getName().substring(0, rule.getMaxLength()), column.getType()); } catch (IndexOutOfBoundsException ex) { logger.error("FATAL unexpected error in \"truncate\" rule: substring failed."); throw new AssertionError("FATAL unexpected error in \"truncate\" rule: substring failed.", ex); } } } return builder.build(); }
public static TimestampFormatter[] newTimestampColumnFormatters( TimestampFormatter.Task formatterTask, Schema schema, Map<String, ? extends TimestampFormatter.TimestampColumnOption> columnOptions) { TimestampFormatter[] formatters = new TimestampFormatter[schema.getColumnCount()]; int i = 0; for (Column column : schema.getColumns()) { if (column.getType() instanceof TimestampType) { Optional<TimestampFormatter.TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName())); formatters[i] = TimestampFormatter.of(formatterTask, option); } i++; } return formatters; } }
@Override public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) { final PluginTask task = config.loadConfig(PluginTask.class); if (task.getPrintsColumnNames()) { for (final Column column : schema.getColumns()) { if (column.getIndex() > 0) { System.out.print(","); } System.out.print(column.getName()); } System.out.println(""); } return resume(task.dump(), schema, taskCount, control); }
private Schema applyLowerToUpperRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toUpperCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
private Schema applyUpperToLowerRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toLowerCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
String name = column.getName(); if (name.matches(regexBuilder.toString())) { if (replace.isPresent()) {
schemaBuilder.add(column.getName().replaceAll(regexBuilder.toString(), replace), column.getType());
private void writeHeader(Schema schema, LineEncoder encoder, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString) { String delimiterString = String.valueOf(delimiter); for (Column column : schema.getColumns()) { if (column.getIndex() != 0) { encoder.addText(delimiterString); } encoder.addText(setEscapeAndQuoteValue(column.getName(), delimiter, policy, quote, escape, newline, nullString)); } encoder.addNewLine(); }
List<String> removeColumns = getExistentColumns(inputSchema, task.getRemove().get(), acceptUnmatchedColumns); for (Column column : inputSchema.getColumns()) { if (!removeColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType())); indexMapping[column.getIndex()] = index; index++; List<String> keepColumns = getExistentColumns(inputSchema, task.getKeep().get(), acceptUnmatchedColumns); for (Column column : inputSchema.getColumns()) { if (keepColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType())); indexMapping[column.getIndex()] = index; index++;
private Schema applyRegexReplaceRule(Schema inputSchema, RegexReplaceRule rule) { final String match = rule.getMatch(); final String replace = rule.getReplace(); Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { // TODO(dmikurube): Check if we need a kind of sanitization? try { builder.add(column.getName().replaceAll(match, replace), column.getType()); } catch (PatternSyntaxException ex) { throw new ConfigException(ex); } } return builder.build(); }
@Override public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control) { PluginTask task = config.loadConfig(PluginTask.class); Map<String, String> renameMap = task.getRenameMap(); List<ConfigSource> rulesList = task.getRulesList(); // Check if the given column in "columns" exists or not. for (String columnName : renameMap.keySet()) { inputSchema.lookupColumn(columnName); // throws SchemaConfigException } // Rename by "columns": to be applied before "rules". Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { String name = column.getName(); if (renameMap.containsKey(name)) { name = renameMap.get(name); } builder.add(name, column.getType()); } Schema intermediateSchema = builder.build(); // Rename by "rules". Schema outputSchema = intermediateSchema; for (ConfigSource rule : rulesList) { outputSchema = applyRule(rule, intermediateSchema); intermediateSchema = outputSchema; } control.run(task.dump(), outputSchema); }
@Override public void run(TaskSource task, Schema newSchema) { ArrayList<String> resolvedColumnNamesList = new ArrayList<>(newSchema.size()); for (Column resolvedColumn : newSchema.getColumns()) { resolvedColumnNamesList.add(resolvedColumn.getName()); } String[] resolvedColumnNames = Iterables.toArray(resolvedColumnNamesList, String.class); assertEquals(expectedColumnNames, resolvedColumnNames); for (int i = 0; i < expectedColumnNames.length; ++i) { Column original = originalSchema.getColumn(i); Column resolved = newSchema.getColumn(i); assertEquals(original.getType(), resolved.getType()); } } });
@Override public void run(TaskSource task, Schema newSchema) { // _c0 -> _c0_new Column old0 = SCHEMA.getColumn(0); Column new0 = newSchema.getColumn(0); assertEquals("_c0_new", new0.getName()); assertEquals(old0.getType(), new0.getType()); // _c1 is not changed Column old1 = SCHEMA.getColumn(1); Column new1 = newSchema.getColumn(1); assertEquals("_c1", new1.getName()); assertEquals(old1.getType(), new1.getType()); } });