/** * check if the current schema version in metastore matches the Hive version */ @VisibleForTesting void verifySchemaVersion() throws HiveMetaException { // don't check version if its a dry run if (dryRun) { return; } String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false)); // verify that the new version is added to schema assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion); }
String hiveSchemaVer = metastoreSchemaInfo.getHiveSchemaVersion(); if (metastoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) { LOG.debug("Found expected HMS version of {}", dbSchemaVer); } else {
try { MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false); version = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); } catch (HiveMetaException he) { System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage()); List<String> subScripts = new ArrayList<>(); String baseDir = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir()).getParent(); String schemaFile = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(), schemaTool.getMetaStoreSchemaInfo().generateInitFileName(version)).getPath(); try { LOG.debug("Parsing schema script " + schemaFile);
@Override void execute() throws HiveMetaException { schemaTool.testConnectionToMetastore(); System.out.println("Starting user creation"); String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(); String protoCreateFile = schemaTool.getMetaStoreSchemaInfo().getCreateUserScript(); try { File createFile = subUserAndPassword(scriptDir, protoCreateFile); System.out.println("Creation script " + createFile.getAbsolutePath()); if (!schemaTool.isDryRun()) { if ("oracle".equals(schemaTool.getDbType())) oracleCreateUserHack(createFile); else schemaTool.execSql(createFile.getParent(), createFile.getName()); System.out.println("User creation completed"); } } catch (IOException e) { throw new HiveMetaException("User creation FAILED!" + " Metastore unusable !!", e); } }
@Override void execute() throws HiveMetaException { ensureFromVersion(); if (schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion().equals(fromVersion)) { System.out.println("No schema upgrade required from version " + fromVersion); return; } // Find the list of scripts to execute for this upgrade List<String> upgradeScripts = schemaTool.getMetaStoreSchemaInfo().getUpgradeScripts(fromVersion); schemaTool.testConnectionToMetastore(); System.out.println("Starting upgrade metastore schema from version " + fromVersion + " to " + schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion()); String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(); try { for (String scriptFile : upgradeScripts) { System.out.println("Upgrade script " + scriptFile); if (!schemaTool.isDryRun()) { runPreUpgrade(scriptDir, scriptFile); schemaTool.execSql(scriptDir, scriptFile); System.out.println("Completed " + scriptFile); } } } catch (IOException e) { throw new HiveMetaException("Upgrade FAILED! Metastore state would be inconsistent !!", e); } // Revalidated the new version after upgrade schemaTool.verifySchemaVersion(); }
private void ensureToVersion() throws HiveMetaException { if (toVersion != null) { return; } // If null then current hive version is used toVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion(); System.out.println("Initializing the schema to: " + toVersion); }
protected void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion) throws HiveMetaException { if (!metaStoreSchemaInfo.isVersionCompatible(hiveSchemaVersion, dbSchemaVersion)) { throw new HiveMetaException("Metastore schema version is not compatible. Hive Version: " + hiveSchemaVersion + ", Database Schema Version: " + dbSchemaVersion); } }
@Override void execute() throws HiveMetaException { ensureToVersion(); schemaTool.testConnectionToMetastore(); System.out.println("Starting metastore schema initialization to " + toVersion); String initScriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(); String initScriptFile = schemaTool.getMetaStoreSchemaInfo().generateInitFileName(toVersion); try { System.out.println("Initialization script " + initScriptFile); if (!schemaTool.isDryRun()) { schemaTool.execSql(initScriptDir, initScriptFile); System.out.println("Initialization script completed"); } } catch (IOException e) { throw new HiveMetaException("Schema initialization FAILED! Metastore state would be inconsistent!", e); } if (validate) { schemaTool.verifySchemaVersion(); } } }
private void ensureFromVersion() throws HiveMetaException { if (fromVersion != null) { return; } // If null, then read from the metastore MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false); fromVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); if (fromVersion == null || fromVersion.isEmpty()) { throw new HiveMetaException("Schema version not stored in the metastore. " + "Metastore schema is too old or corrupt. Try specifying the version manually"); } System.out.println("Upgrading from the version " + fromVersion); }
/** * Run pre-upgrade scripts corresponding to a given upgrade script, * if any exist. The errors from pre-upgrade are ignored. * Pre-upgrade scripts typically contain setup statements which * may fail on some database versions and failure is ignorable. * * @param scriptDir upgrade script directory name * @param scriptFile upgrade script file name */ private void runPreUpgrade(String scriptDir, String scriptFile) { for (int i = 0;; i++) { String preUpgradeScript = schemaTool.getMetaStoreSchemaInfo().getPreUpgradeScriptName(i, scriptFile); File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript); if (!preUpgradeScriptFile.isFile()) { break; } try { schemaTool.execSql(scriptDir, preUpgradeScript); System.out.println("Completed " + preUpgradeScript); } catch (Exception e) { // Ignore the pre-upgrade script errors System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + e.getMessage()); if (schemaTool.isVerbose()) { e.printStackTrace(); } } } } }
if (metaStoreSchemaInfo.getHiveSchemaVersion().equals(fromSchemaVer)) { System.out.println("No schema upgrade required from version " + fromSchemaVer); return; metaStoreSchemaInfo.getUpgradeScripts(fromSchemaVer); testConnectionToMetastore(); System.out.println("Starting upgrade metastore schema from version " + fromSchemaVer + " to " + metaStoreSchemaInfo.getHiveSchemaVersion()); String scriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir(); try { for (String scriptFile : upgradeScripts) {
if (e.getCause() instanceof MissingTableException) { throw new MetaException("Version table not found. " + "The metastore is not upgraded to " + MetaStoreSchemaInfoFactory.get(getConf()).getHiveSchemaVersion()); } else { throw e;
@Test public void testIsVersionCompatible() throws Exception { // first argument is hiveVersion, it is compatible if 2nd argument - dbVersion is // greater than or equal to it // check the compatible case IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(MetastoreConf.newMetastoreConf()); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.1")); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.2")); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("1.0.2", "2.0.1")); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.9", "9.0.0")); // check equivalent versions, should be compatible Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.0", "0.13.1")); Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.1", "0.13.0")); // check incompatible versions Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("0.1.1", "0.1.0")); Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("4.0.1", "0.1.0")); }
/** * Initialize the metastore schema * * @param toVersion * If null then current hive version is used * @throws MetaException */ public void doInit(String toVersion) throws HiveMetaException { testConnectionToMetastore(); System.out.println("Starting metastore schema initialization to " + toVersion); String initScriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir(); String initScriptFile = metaStoreSchemaInfo.generateInitFileName(toVersion); try { System.out.println("Initialization script " + initScriptFile); if (!dryRun) { runBeeLine(initScriptDir, initScriptFile); System.out.println("Initialization script completed"); } } catch (IOException e) { throw new HiveMetaException("Schema initialization FAILED!" + " Metastore state would be inconsistent !!", e); } }
@Override void execute() throws HiveMetaException { HiveSchemaHelper.MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(true); String dbVersion = null; try { dbVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); } catch (HiveMetaException e) { LOG.info("Exception getting db version:" + e.getMessage()); LOG.info("Try to initialize db schema"); } SchemaToolTask task; if (dbVersion == null) { task = new SchemaToolTaskInit(); } else { task = new SchemaToolTaskUpgrade(); } task.setHiveSchemaTool(schemaTool); task.setCommandLineArguments(cl); task.execute(); } }
/** * Run pre-upgrade scripts corresponding to a given upgrade script, * if any exist. The errors from pre-upgrade are ignored. * Pre-upgrade scripts typically contain setup statements which * may fail on some database versions and failure is ignorable. * * @param scriptDir upgrade script directory name * @param scriptFile upgrade script file name */ private void runPreUpgrade(String scriptDir, String scriptFile) { for (int i = 0;; i++) { String preUpgradeScript = metaStoreSchemaInfo.getPreUpgradeScriptName(i, scriptFile); File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript); if (!preUpgradeScriptFile.isFile()) { break; } try { runBeeLine(scriptDir, preUpgradeScript); System.out.println("Completed " + preUpgradeScript); } catch (Exception e) { // Ignore the pre-upgrade script errors System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + e.getMessage()); if (verbose) { e.printStackTrace(); } } } }
@Override void execute() throws HiveMetaException { String hiveVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion(); MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(true); String dbVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo); System.out.println("Hive distribution version:\t " + hiveVersion); System.out.println("Metastore schema version:\t " + dbVersion); schemaTool.assertCompatibleVersion(hiveVersion, dbVersion); } }
String hiveSchemaVer = metastoreSchemaInfo.getHiveSchemaVersion(); if (metastoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) { LOG.debug("Found expected HMS version of {}", dbSchemaVer); } else {
/** * Test schema initialization */ @Test public void testSchemaInit() throws Exception { IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf, System.getProperty("test.tmp.dir", "target/tmp"), "derby"); execute(new SchemaToolTaskInit(), "-initSchemaTo " + metastoreSchemaInfo.getHiveSchemaVersion()); schemaTool.verifySchemaVersion(); }
version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false)); } catch (HiveMetaException he) { System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage()); String baseDir = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent(); String schemaFile = new File(metaStoreSchemaInfo.getMetaStoreScriptDir(), metaStoreSchemaInfo.generateInitFileName(version)).getPath(); try { LOG.debug("Parsing schema script " + schemaFile);