static String shouldTableBeExternal(Table tableObj, String ownerName, Configuration conf, IMetaStoreClient hms, Boolean isPathOwnedByHive) throws IOException, MetaException, TException { if (MetaStoreUtils.isNonNativeTable(tableObj)) { return "Table is a non-native (StorageHandler) table"; } if (HiveStrictManagedUtils.isAvroTableWithExternalSchema(tableObj)) { return "Table is an Avro table with an external schema url"; } // List bucketed table cannot be converted to transactional if (HiveStrictManagedUtils.isListBucketedTable(tableObj)) { return "Table is a list bucketed table"; } // If any table/partition directory is not owned by hive, // then assume table is using storage-based auth - set external. // Transactional tables should still remain transactional, // but we should have already checked for that before this point. if (isPathOwnedByHive != null) { // for replication flow, the path ownership must be verified at source cluster itself. return isPathOwnedByHive ? null : String.format("One or more table directories is not owned by hive or non-HDFS path at source cluster"); } else if (shouldTablePathBeExternal(tableObj, ownerName, conf, hms)) { return String.format("One or more table directories not owned by %s, or non-HDFS path", ownerName); } return null; }
public static void validateStrictManagedTableWithThrow(Configuration conf, Table table) throws MetaException { String reason = validateStrictManagedTable(conf, table); if (reason != null) { throw new MetaException(reason); } }
/** * Checks if the table is valid based on the rules for strict managed tables. * @param conf * @param table * @return Null if the table is valid, otherwise a string message indicating why the table is invalid. */ public static String validateStrictManagedTable(Configuration conf, Table table) { if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STRICT_MANAGED_TABLES)) { if (table.isTemporary()) { // temp tables exempted from checks. return null; } TableType tableType = TableType.valueOf(table.getTableType()); if (tableType == TableType.MANAGED_TABLE) { if (!MetaStoreServerUtils.isTransactionalTable(table.getParameters())) { return createValidationError(table, "Table is marked as a managed table but is not transactional."); } if (MetaStoreUtils.isNonNativeTable(table)) { return createValidationError(table, "Table is marked as a managed table but is non-native."); } if (isAvroTableWithExternalSchema(table)) { return createValidationError(table, "Managed Avro table has externally defined schema."); } } } // Table is valid return null; }
private void handle(PreAlterTableEvent context) throws MetaException { if (supportedCatalogs.contains(getTableCatalog(context.getNewTable()))) { handleAlterTableTransactionalProp(context); HiveStrictManagedUtils.validateStrictManagedTableWithThrow(getConf(), context.getNewTable()); } }
private void handle(PreCreateTableEvent context) throws MetaException { if (supportedCatalogs.contains(getTableCatalog(context.getTable()))) { handleCreateTableTransactionalProp(context); HiveStrictManagedUtils.validateStrictManagedTableWithThrow(getConf(), context.getTable()); } }
return false; if (HiveStrictManagedUtils.isAvroTableWithExternalSchema(tableObj)) { String msg = createManagedConversionExcuse(tableObj, "Table is an Avro table with an external schema url"); if (HiveStrictManagedUtils.isListBucketedTable(tableObj)) { String msg = createManagedConversionExcuse(tableObj, "Table is a list bucketed table");
public static boolean migrateTable(Table tableObj, TableType tableType, TableMigrationOption migrationOption, boolean dryRun, HiveUpdater hiveUpdater, IMetaStoreClient hms, Configuration conf) throws HiveException, IOException, TException { switch (migrationOption) { case EXTERNAL: migrateToExternalTable(tableObj, tableType, dryRun, hiveUpdater); break; case MANAGED: migrateToManagedTable(tableObj, tableType, dryRun, hiveUpdater, hms, conf); break; case NONE: break; case VALIDATE: // Check that the table is valid under strict managed tables mode. String reason = HiveStrictManagedUtils.validateStrictManagedTable(conf, tableObj); if (reason != null) { LOG.warn(reason); return true; } break; default: throw new IllegalArgumentException("Unexpected table migration option " + migrationOption); } return false; }