@Override public void shutdown() { rawStore.shutdown(); }
private void updateTablePartitionColStats(RawStore rawStore, String catName, String dbName, String tblName) { boolean committed = false; rawStore.openTransaction(); try { Table table = rawStore.getTable(catName, dbName, tblName); List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); List<String> partNames = rawStore.listPartitionNames(catName, dbName, tblName, (short) -1); // Get partition column stats for this table Deadline.startTimer("getPartitionColumnStatistics"); List<ColumnStatistics> partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); List<Partition> parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); // Also save partitions for consistency as they have the stats state. for (Partition part : parts) { sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); } committed = rawStore.commitTransaction(); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); } finally { if (!committed) { sharedCache.removeAllPartitionColStatsFromCache(catName, dbName, tblName); rawStore.rollbackTransaction(); } } }
try { Deadline.registerIfNot(100000); List<Function> functions = store.getAllFunctions(DEFAULT_CATALOG_NAME); for (Function func : functions) { store.dropFunction(DEFAULT_CATALOG_NAME, func.getDbName(), func.getFunctionName()); for (String catName : store.getCatalogs()) { List<String> dbs = store.getAllDatabases(catName); for (String db : dbs) { List<String> tbls = store.getAllTables(DEFAULT_CATALOG_NAME, db); for (String tbl : tbls) { Deadline.startTimer("getPartition"); List<Partition> parts = store.getPartitions(DEFAULT_CATALOG_NAME, db, tbl, 100); for (Partition part : parts) { store.dropPartition(DEFAULT_CATALOG_NAME, db, tbl, part.getValues()); List<SQLPrimaryKey> pk = store.getPrimaryKeys(DEFAULT_CATALOG_NAME, db, tbl); if (pk != null) { for (SQLPrimaryKey pkcol : pk) { List<SQLForeignKey> fks = store.getForeignKeys(DEFAULT_CATALOG_NAME, null, null, db, tbl); if (fks != null) { for (SQLForeignKey fkcol : fks) { store.dropConstraint(DEFAULT_CATALOG_NAME, db, tbl, constraint); store.dropTable(DEFAULT_CATALOG_NAME, db, tbl); store.dropDatabase(catName, db);
public static RawStore getMSForConf(Configuration conf) throws MetaException { RawStore ms = threadLocalMS.get(); if (ms == null) { ms = newRawStoreForConf(conf); try { ms.verifySchema(); } catch (MetaException e) { ms.shutdown(); throw e; } threadLocalMS.set(ms); ms = threadLocalMS.get(); LOG.info("Created RawStore: " + ms + " from thread id: " + Thread.currentThread().getId()); } return ms; }
@Override public void add_runtime_stats(RuntimeStat stat) throws TException { startFunction("store_runtime_stats"); Exception ex = null; boolean success = false; RawStore ms = getMS(); try { ms.openTransaction(); ms.addRuntimeStat(stat); success = ms.commitTransaction(); } catch (Exception e) { LOG.error("Caught exception", e); ex = e; throw e; } finally { if (!success) { ms.rollbackTransaction(); } endFunction("store_runtime_stats", success, ex); } }
@Override public void alter_function(String dbName, String funcName, Function newFunc) throws TException { String[] parsedDbName = parseDbName(dbName, conf); validateForAlterFunction(parsedDbName[DB_NAME], funcName, newFunc); boolean success = false; RawStore ms = getMS(); try { ms.openTransaction(); ms.alterFunction(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], funcName, newFunc); success = ms.commitTransaction(); } catch (InvalidObjectException e) { // Throwing MetaException instead of InvalidObjectException as the InvalidObjectException // is not defined for the alter_function method in the Thrift interface. throwMetaException(e); } finally { if (!success) { ms.rollbackTransaction(); } } }
ms.openTransaction(); part = ms.getPartition(catName, db_name, tbl_name, part_vals); tbl = get_table_core(catName, db_name, tbl_name, null); tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(tbl, deleteData); if (!ms.dropPartition(catName, db_name, tbl_name, part_vals)) { throw new MetaException("Unable to drop partition"); } else { envContext); isSourceOfReplication = ReplChangeManager.isSourceOfReplication(ms.getDatabase(catName, db_name)); success = ms.commitTransaction(); ms.rollbackTransaction(); } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) { if (tableDataShouldBeDeleted) {
ms.openTransaction(); db = ms.getDatabase(catName, name); materializedViews = ms.getTableObjectsByName(catName, name, materializedViewNames.subList(startIndex, endIndex)); } catch (UnknownDBException e) { throw new MetaException(e.getMessage()); tables = ms.getTableObjectsByName(catName, name, allTables.subList(startIndex, endIndex)); } catch (UnknownDBException e) { throw new MetaException(e.getMessage()); if (ms.dropDatabase(catName, name)) { if (!transactionalListeners.isEmpty()) { transactionalListenerResponses = success = ms.commitTransaction(); ms.rollbackTransaction(); } else if (deleteData) {
cat = getMS().getCatalog(db.getCatalogName()); } catch (NoSuchObjectException e) { LOG.error("No such catalog " + db.getCatalogName()); ms.openTransaction(); ms.createDatabase(db); success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction(); if (madeDir) { wh.deleteDir(dbPath, true, db);
boolean pathCreated = false; RawStore ms = getMS(); ms.openTransaction(); ms.getTable( parsedDestDbName[CAT_NAME], parsedDestDbName[DB_NAME], destTableName, null); if (destinationTable == null) { ms.getTable( parsedSourceDbName[CAT_NAME], parsedSourceDbName[DB_NAME], sourceTableName, null); if (sourceTable == null) { List<String> destPartitionNames = ms.listPartitionNames(parsedDestDbName[CAT_NAME], parsedDestDbName[DB_NAME], destTableName, (short) -1); if (destPartitionNames != null && !destPartitionNames.isEmpty()) { Database srcDb = ms.getDatabase(parsedSourceDbName[CAT_NAME], parsedSourceDbName[DB_NAME]); Database destDb = ms.getDatabase(parsedDestDbName[CAT_NAME], parsedDestDbName[DB_NAME]); if (!isRenameAllowed(srcDb, destDb)) { throw new MetaException("Exchange partition not allowed for " + Warehouse.makePartName(destinationTable.getPartitionKeys(), partition.getValues())); destPartition.getSd().setLocation(destPartitionPath.toString()); ms.addPartition(destPartition); destPartitions.add(destPartition); ms.dropPartition(parsedSourceDbName[CAT_NAME], partition.getDbName(), sourceTable.getTableName(), partition.getValues()); success = ms.commitTransaction();
msdb.openTransaction(); Table tbl = msdb.getTable(catName, dbname, name, null); if (tbl == null) { throw new InvalidObjectException( "Unable to alter partition because table or database does not exist."); oldPart = msdb.getPartition(catName, dbname, name, new_part.getValues()); if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { msdb.alterPartition( catName, dbname, name, new_part.getValues(), new_part, validWriteIds); if (transactionalListeners != null && !transactionalListeners.isEmpty()) { success = msdb.commitTransaction(); } catch (InvalidObjectException e) { LOG.warn("Alter failed", e); } finally { if(!success) { msdb.rollbackTransaction(); Database db; try { msdb.openTransaction(); Table tbl = msdb.getTable(DEFAULT_CATALOG_NAME, dbname, name, null); if (tbl == null) { throw new InvalidObjectException(
msdb.openTransaction(); Table tbl = msdb.getTable(catName, dbname, name, null); if (tbl == null) { throw new InvalidObjectException( Partition oldTmpPart = msdb.getPartition(catName, dbname, name, tmpPart.getValues()); oldParts.add(oldTmpPart); partValsList.add(tmpPart.getValues()); msdb.alterPartitions(catName, dbname, name, partValsList, new_parts, writeId, writeIdList); Iterator<Partition> oldPartsIt = oldParts.iterator(); for (Partition newPart : new_parts) { success = msdb.commitTransaction(); } catch (InvalidObjectException | NoSuchObjectException e) { throw new InvalidOperationException("Alter partition operation failed: " + e); } finally { if(!success) { msdb.rollbackTransaction();
Database db = null; try { ms.openTransaction(); part.setCatName(catName); part.setDbName(dbName); tbl = ms.getTable(part.getCatName(), part.getDbName(), part.getTableName(), null); if (tbl == null) { throw new InvalidObjectException( old_part = ms.getPartition(part.getCatName(), part.getDbName(), part .getTableName(), part.getValues()); } catch (NoSuchObjectException e) { if (ms.addPartition(part)) { if (!transactionalListeners.isEmpty()) { transactionalListenerResponses = success = ms.commitTransaction(); ms.rollbackTransaction(); if (madeDir) { wh.deleteDir(partLocation, true, db);
boolean pathCreated = false; RawStore ms = getMS(); ms.openTransaction(); Table destinationTable = ms.getTable(destDbName, destTableName); Table sourceTable = ms.getTable(sourceDbName, sourceTableName); List<String> partVals = MetaStoreUtils.getPvals(sourceTable.getPartitionKeys(), partitionSpecs); Warehouse.makePartName(destinationTable.getPartitionKeys(), partition.getValues())); destPartition.getSd().setLocation(destPartitionPath.toString()); ms.addPartition(destPartition); ms.dropPartition(partition.getDbName(), sourceTable.getTableName(), partition.getValues()); success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction(); if (pathCreated) { wh.renameDir(destPath, sourcePath);
private static void dropAllStoreObjects(RawStore store) throws MetaException, InvalidObjectException, InvalidInputException { try { Deadline.registerIfNot(100000); Deadline.startTimer("getPartition"); List<String> dbs = store.getAllDatabases(DEFAULT_CATALOG_NAME); for (int i = 0; i < dbs.size(); i++) { String db = dbs.get(i); List<String> tbls = store.getAllTables(DEFAULT_CATALOG_NAME, db); for (String tbl : tbls) { List<Partition> parts = store.getPartitions(DEFAULT_CATALOG_NAME, db, tbl, 100); for (Partition part : parts) { store.dropPartition(DEFAULT_CATALOG_NAME, db, tbl, part.getValues()); } store.dropTable(DEFAULT_CATALOG_NAME, db, tbl); } store.dropDatabase(DEFAULT_CATALOG_NAME, db); } } catch (NoSuchObjectException e) { } }
private void updateTableColStats(RawStore rawStore, String catName, String dbName, String tblName) { boolean committed = false; rawStore.openTransaction(); try { Table table = rawStore.getTable(catName, dbName, tblName); if (!table.isSetPartitionKeys()) { List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); ColumnStatistics tableColStats = rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer(); if (tableColStats != null) { sharedCache.refreshTableColStatsInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), tableColStats.getStatsObj()); // Update the table to get consistent stats state. sharedCache.alterTableInCache(catName, dbName, tblName, table); } } committed = rawStore.commitTransaction(); } catch (MetaException | NoSuchObjectException e) { LOG.info("Unable to refresh table column stats for table: " + tblName, e); } finally { if (!committed) { sharedCache.removeAllTableColStatsFromCache(catName, dbName, tblName); rawStore.rollbackTransaction(); } } }
getMS().openTransaction(); try { Table table = getMS().getTable(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName); ret = getMS().deleteTableColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colName); if (ret) { if (transactionalListeners != null && !transactionalListeners.isEmpty()) { committed = getMS().commitTransaction(); } finally { if (!committed) { getMS().rollbackTransaction();
getMS().openTransaction(); boolean committed = false; try { parameters = getMS().updateTableColumnStatistics(colStats, validWriteIds, writeId); if (parameters != null) { Table tableObj = getMS().getTable(colStats.getStatsDesc().getCatName(), colStats.getStatsDesc().getDbName(), colStats.getStatsDesc().getTableName(), validWriteIds); committed = getMS().commitTransaction(); } finally { if (!committed) { getMS().rollbackTransaction();
private void create_type_core(final RawStore ms, final Type type) throws AlreadyExistsException, MetaException, InvalidObjectException { if (!MetaStoreUtils.validateName(type.getName(), null)) { throw new InvalidObjectException("Invalid type name"); } boolean success = false; try { ms.openTransaction(); if (is_type_exists(ms, type.getName())) { throw new AlreadyExistsException("Type " + type.getName() + " already exists"); } ms.createType(type); success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction(); } } }
RawStore ms = getMS(); try { SchemaVersion schemaVersion = ms.getSchemaVersion(version); if (schemaVersion == null) { throw new NoSuchObjectException("No schema version " + version); ms.openTransaction(); try { ms.dropSchemaVersion(version); if (!transactionalListeners.isEmpty()) { transactionalListenersResponses = EventType.DROP_SCHEMA_VERSION, new DropSchemaVersionEvent(true, this, schemaVersion)); success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction();