public int getFixedStorageSize() { int total = 0; for (Column column : columns) { total += column.getType().getFixedStorageSize(); } return total; }
public Type getColumnType(int index) { return getColumn(index).getType(); }
@Override public String toString() { StringBuilder sbuf = new StringBuilder(); sbuf.append("Schema{\n"); for (Column c : columns) { sbuf.append(String.format(" %4d: %s %s%n", c.getIndex(), c.getName(), c.getType())); } sbuf.append("}"); return sbuf.toString(); } }
@Override public String toString() { return String.format("Column{index:%d, name:%s, type:%s}", getIndex(), getName(), getType().getName()); } }
public static TimestampFormatter[] newTimestampColumnFormatters( TimestampFormatter.Task formatterTask, Schema schema, Map<String, ? extends TimestampFormatter.TimestampColumnOption> columnOptions) { TimestampFormatter[] formatters = new TimestampFormatter[schema.getColumnCount()]; int i = 0; for (Column column : schema.getColumns()) { if (column.getType() instanceof TimestampType) { Optional<TimestampFormatter.TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName())); formatters[i] = TimestampFormatter.of(formatterTask, option); } i++; } return formatters; } }
private static ColumnValue newValue(Column column) { Type type = column.getType(); if (type.equals(Types.BOOLEAN)) { return new BooleanColumnValue(column); } else if (type.equals(Types.DOUBLE)) { return new DoubleColumnValue(column); } else if (type.equals(Types.LONG)) { return new LongColumnValue(column); } else if (type.equals(Types.STRING)) { return new StringColumnValue(column); } else if (type.equals(Types.JSON)) { return new JsonColumnValue(column); } else if (type.equals(Types.TIMESTAMP)) { return new TimestampColumnValue(column); } else { throw new IllegalStateException("Unsupported type " + type.getName()); } }
private Schema applyLowerToUpperRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toUpperCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
schemaBuidler.add(name, column.getType());
private Schema applyTruncateRule(Schema inputSchema, TruncateRule rule) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { if (column.getName().length() <= rule.getMaxLength()) { builder.add(column.getName(), column.getType()); } else { try { builder.add(column.getName().substring(0, rule.getMaxLength()), column.getType()); } catch (IndexOutOfBoundsException ex) { logger.error("FATAL unexpected error in \"truncate\" rule: substring failed."); throw new AssertionError("FATAL unexpected error in \"truncate\" rule: substring failed.", ex); } } } return builder.build(); }
private Schema applyUpperToLowerRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toLowerCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
schemaBuilder.add(column.getName().replaceAll(regexBuilder.toString(), replace), column.getType());
@Override public void run(TaskSource task, Schema newSchema) { ArrayList<String> resolvedColumnNamesList = new ArrayList<>(newSchema.size()); for (Column resolvedColumn : newSchema.getColumns()) { resolvedColumnNamesList.add(resolvedColumn.getName()); } String[] resolvedColumnNames = Iterables.toArray(resolvedColumnNamesList, String.class); assertEquals(expectedColumnNames, resolvedColumnNames); for (int i = 0; i < expectedColumnNames.length; ++i) { Column original = originalSchema.getColumn(i); Column resolved = newSchema.getColumn(i); assertEquals(original.getType(), resolved.getType()); } } });
private Schema applyRegexReplaceRule(Schema inputSchema, RegexReplaceRule rule) { final String match = rule.getMatch(); final String replace = rule.getReplace(); Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { // TODO(dmikurube): Check if we need a kind of sanitization? try { builder.add(column.getName().replaceAll(match, replace), column.getType()); } catch (PatternSyntaxException ex) { throw new ConfigException(ex); } } return builder.build(); }
@Override public void run(TaskSource task, Schema newSchema) { // _c0 -> _c0_new Column old0 = SCHEMA.getColumn(0); Column new0 = newSchema.getColumn(0); assertEquals("_c0_new", new0.getName()); assertEquals(old0.getType(), new0.getType()); // _c1 is not changed Column old1 = SCHEMA.getColumn(1); Column new1 = newSchema.getColumn(1); assertEquals("_c1", new1.getName()); assertEquals(old1.getType(), new1.getType()); } });
for (Column column : inputSchema.getColumns()) { if (!removeColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType())); indexMapping[column.getIndex()] = index; index++; for (Column column : inputSchema.getColumns()) { if (keepColumns.contains(column.getName())) { outputColumns.add(new Column(index, column.getName(), column.getType())); indexMapping[column.getIndex()] = index; index++;
@Override public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control) { PluginTask task = config.loadConfig(PluginTask.class); Map<String, String> renameMap = task.getRenameMap(); List<ConfigSource> rulesList = task.getRulesList(); // Check if the given column in "columns" exists or not. for (String columnName : renameMap.keySet()) { inputSchema.lookupColumn(columnName); // throws SchemaConfigException } // Rename by "columns": to be applied before "rules". Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { String name = column.getName(); if (renameMap.containsKey(name)) { name = renameMap.get(name); } builder.add(name, column.getType()); } Schema intermediateSchema = builder.build(); // Rename by "rules". Schema outputSchema = intermediateSchema; for (ConfigSource rule : rulesList) { outputSchema = applyRule(rule, intermediateSchema); intermediateSchema = outputSchema; } control.run(task.dump(), outputSchema); }
public DynamicColumnSetter newColumnSetter(PageBuilder pageBuilder, Column column) { Type type = column.getType(); if (type instanceof BooleanType) { return new BooleanColumnSetter(pageBuilder, column, defaultValue);
if (buffer != null) { for (Column column : schema.getColumns()) { Type type = column.getType(); switch (type.getName()) { case "boolean":
private void buildOutputDefaultMap() { for (Column outputColumn : outputSchema.getColumns()) { String name = outputColumn.getName(); Type type = outputColumn.getType(); Object defaultValue = getDefault(task, name, type, task.getColumns()); if (defaultValue == null) { defaultValue = getDefault(task, name, type, task.getAddColumns()); } if (defaultValue != null) { outputDefaultMap.put(outputColumn, defaultValue); } } }