(tableFilter != null && !tableFilter.isIncluded(tableId))) { continue;
@Test public void shouldUseFiltersForAlterTable() { parser = new MysqlDdlParserWithSimpleTestListener(listener, TableFilter.fromPredicate(x -> !x.table().contains("ignored"))); final String ddl = "CREATE TABLE ok (id int primary key, val smallint);" + System.lineSeparator() + "ALTER TABLE ignored ADD COLUMN(x tinyint)" + System.lineSeparator() + "ALTER TABLE ok ADD COLUMN(y tinyint)"; parser.parse(ddl, tables); assertThat(((MysqlDdlParserWithSimpleTestListener)parser).getParsingExceptionsFromWalker()).isEmpty(); assertThat(tables.size()).isEqualTo(1); final Table t1 = tables.forTable(null, null, "ok"); assertThat(t1.columns()).hasSize(3); final Column c1 = t1.columns().get(0); final Column c2 = t1.columns().get(1); final Column c3 = t1.columns().get(2); assertThat(c1.name()).isEqualTo("id"); assertThat(c1.typeName()).isEqualTo("INT"); assertThat(c2.name()).isEqualTo("val"); assertThat(c2.typeName()).isEqualTo("SMALLINT"); assertThat(c3.name()).isEqualTo("y"); assertThat(c3.typeName()).isEqualTo("TINYINT"); }
(tableFilter != null && !tableFilter.isIncluded(tableId))) { continue;
if (tableFilter.isIncluded(tableIdWithCatalog)) { TableEditor editor = tables.editTable(tableId); editor.tableId(tableIdWithCatalog);
/** * @param config the configuration; may not be null */ public Filters(PostgresConnectorConfig config) { // we always want to exclude PG system schemas as they are never part of logical decoding events String schemaBlacklist = config.schemaBlacklist(); if (schemaBlacklist != null) { schemaBlacklist = schemaBlacklist + "," + SYSTEM_SCHEMA_BLACKLIST; } else { schemaBlacklist = SYSTEM_SCHEMA_BLACKLIST; } String tableBlacklist = config.tableBlacklist(); if (tableBlacklist != null) { tableBlacklist = tableBlacklist + "," + TEMP_TABLE_BLACKLIST; } else { tableBlacklist = TEMP_TABLE_BLACKLIST; } // Define the filter using the whitelists and blacklists for table names ... this.tableFilter = TableFilter.fromPredicate(Selectors.tableSelector() .includeTables(config.tableWhitelist()) .excludeTables(tableBlacklist) .includeSchemas(config.schemaWhitelist()) .excludeSchemas(schemaBlacklist) .build()); // Define the filter that excludes blacklisted columns, truncated columns, and masked columns ... this.columnFilter = Selectors.excludeColumns(config.columnBlacklist()); }
@Test public void shouldUseFiltersForAlterTable() { parser = new MysqlDdlParserWithSimpleTestListener(listener, TableFilter.fromPredicate(x -> !x.table().contains("ignored"))); final String ddl = "CREATE TABLE ok (id int primary key, val smallint);" + System.lineSeparator() + "ALTER TABLE ignored ADD COLUMN(x tinyint)" + System.lineSeparator() + "ALTER TABLE ok ADD COLUMN(y tinyint)"; parser.parse(ddl, tables); assertThat(((MysqlDdlParserWithSimpleTestListener)parser).getParsingExceptionsFromWalker()).isEmpty(); assertThat(tables.size()).isEqualTo(1); final Table t1 = tables.forTable(null, null, "ok"); assertThat(t1.columns()).hasSize(3); final Column c1 = t1.columns().get(0); final Column c2 = t1.columns().get(1); final Column c3 = t1.columns().get(2); assertThat(c1.name()).isEqualTo("id"); assertThat(c1.typeName()).isEqualTo("INT"); assertThat(c2.name()).isEqualTo("val"); assertThat(c2.typeName()).isEqualTo("SMALLINT"); assertThat(c3.name()).isEqualTo("y"); assertThat(c3.typeName()).isEqualTo("TINYINT"); }
@Test @FixFor("DBZ-780") public void shouldRenameColumnWithoutDefinition() { parser = new MysqlDdlParserWithSimpleTestListener(listener, TableFilter.fromPredicate(x -> !x.table().contains("ignored"))); final String ddl = "CREATE TABLE foo (id int primary key, old INT);" + System.lineSeparator() + "ALTER TABLE foo RENAME COLUMN old to new "; parser.parse(ddl, tables); assertThat(((MysqlDdlParserWithSimpleTestListener)parser).getParsingExceptionsFromWalker()).isEmpty(); assertThat(tables.size()).isEqualTo(1); final Table t1 = tables.forTable(null, null, "foo"); assertThat(t1.columns()).hasSize(2); final Column c1 = t1.columns().get(0); final Column c2 = t1.columns().get(1); assertThat(c1.name()).isEqualTo("id"); assertThat(c1.typeName()).isEqualTo("INT"); assertThat(c2.name()).isEqualTo("new"); assertThat(c2.typeName()).isEqualTo("INT"); }
protected boolean isFilteredOut(TableId id) { return !filters.tableFilter().isIncluded(id); }
public MysqlDdlParserWithSimpleTestListener(DdlChanges changesListener, boolean includeViews) { this(changesListener, includeViews, TableFilter.includeAll()); }
private void determineCapturedTables(SnapshotContext ctx) throws Exception { Set<TableId> allTableIds = getAllTableIds(ctx); Set<TableId> capturedTables = new HashSet<>(); for (TableId tableId : allTableIds) { if (connectorConfig.getTableFilters().dataCollectionFilter().isIncluded(tableId)) { LOGGER.trace("Adding table {} to the list of captured tables", tableId); capturedTables.add(tableId); } else { LOGGER.trace("Ignoring table {} as it's not included in the filter configuration", tableId); } } ctx.capturedTables = capturedTables; }
@Test @FixFor("DBZ-903") public void shouldParseFunctionNamedDatabase() { parser = new MysqlDdlParserWithSimpleTestListener(listener, TableFilter.fromPredicate(x -> !x.table().contains("ignored"))); final String ddl = "SELECT `table_name` FROM `information_schema`.`TABLES` WHERE `table_schema` = DATABASE()"; parser.parse(ddl, tables); }
public Tables subset(TableFilter filter) { if (filter == null) return this; return lock.read(() -> { Tables result = new Tables(tableIdCaseInsensitive); tablesByTableId.forEach((tableId, table) -> { if (filter.isIncluded(tableId)) { result.overwriteTable(table); } }); return result; }); }
/** * Get the {@link Table} meta-data for the table with the given identifier, if that table exists and is * included by the filter configuration * * @param id the table identifier; may be null * @return the current table definition, or null if there is no table with the given identifier, if the identifier is null, * or if the table has been excluded by the filters */ public Table tableFor(TableId id) { return tableFilter.isIncluded(id) ? tables.forTable(id) : null; }
/** * Builds up the CDC event schema for the given table and stores it in this schema. */ protected void buildAndRegisterSchema(Table table) { if (tableFilter.isIncluded(table.id())) { TableSchema schema = schemaBuilder.create(schemaPrefix, getEnvelopeSchemaName(table), table, columnFilter, columnMappers); schemasByTableId.put(table.id(), schema); } }
configuration, topicSelector, TableFilter.fromPredicate(tableFilters.tableFilter()), tableFilters.columnFilter(), new TableSchemaBuilder(
if (connectorConfig.getTableFilters().dataCollectionFilter().isIncluded(changeTable.getSourceTableId())) { return true;
/** * @param config the configuration; may not be null */ public Filters(PostgresConnectorConfig config) { // we always want to exclude PG system schemas as they are never part of logical decoding events String schemaBlacklist = config.schemaBlacklist(); if (schemaBlacklist != null) { schemaBlacklist = schemaBlacklist + "," + SYSTEM_SCHEMA_BLACKLIST; } else { schemaBlacklist = SYSTEM_SCHEMA_BLACKLIST; } String tableBlacklist = config.tableBlacklist(); if (tableBlacklist != null) { tableBlacklist = tableBlacklist + "," + TEMP_TABLE_BLACKLIST; } else { tableBlacklist = TEMP_TABLE_BLACKLIST; } // Define the filter using the whitelists and blacklists for table names ... this.tableFilter = TableFilter.fromPredicate(Selectors.tableSelector() .includeTables(config.tableWhitelist()) .excludeTables(tableBlacklist) .includeSchemas(config.schemaWhitelist()) .excludeSchemas(schemaBlacklist) .build()); // Define the filter that excludes blacklisted columns, truncated columns, and masked columns ... this.columnFilter = Selectors.excludeColumns(config.columnBlacklist()); }
@Test @FixFor("DBZ-780") public void shouldRenameColumnWithoutDefinition() { parser = new MysqlDdlParserWithSimpleTestListener(listener, TableFilter.fromPredicate(x -> !x.table().contains("ignored"))); final String ddl = "CREATE TABLE foo (id int primary key, old INT);" + System.lineSeparator() + "ALTER TABLE foo RENAME COLUMN old to new "; parser.parse(ddl, tables); assertThat(((MysqlDdlParserWithSimpleTestListener)parser).getParsingExceptionsFromWalker()).isEmpty(); assertThat(tables.size()).isEqualTo(1); final Table t1 = tables.forTable(null, null, "foo"); assertThat(t1.columns()).hasSize(2); final Column c1 = t1.columns().get(0); final Column c2 = t1.columns().get(1); assertThat(c1.name()).isEqualTo("id"); assertThat(c1.typeName()).isEqualTo("INT"); assertThat(c2.name()).isEqualTo("new"); assertThat(c2.typeName()).isEqualTo("INT"); }
public MySqlAntlrDdlParser() { this(null, TableFilter.includeAll()); }
@Override public void enterAlterTable(MySqlParser.AlterTableContext ctx) { final TableId tableId = parser.parseQualifiedTableId(ctx.tableName().fullId()); if (!parser.getTableFilter().isIncluded(tableId)) { LOG.debug("Ignoring ALTER TABLE statement for non-whitelisted table {}", tableId); return; } tableEditor = parser.databaseTables().editTable(tableId); if (tableEditor == null) { throw new ParsingException(null, "Trying to alter table " + tableId.toString() + ", which does not exist. Query: " + getText(ctx)); } super.enterAlterTable(ctx); }