private List<Path> getLocationsForTruncate(final RawStore ms, final String catName, final String dbName, final String tableName, final Table table, final List<String> partNames) throws Exception { List<Path> locations = new ArrayList<>(); if (partNames == null) { if (0 != table.getPartitionKeysSize()) { for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) { locations.add(new Path(partition.getSd().getLocation())); } } else { locations.add(new Path(table.getSd().getLocation())); } } else { for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { locations.add(new Path(partition.getSd().getLocation())); } } return locations; }
private void updateTablePartitions(RawStore rawStore, String catName, String dbName, String tblName) { try { Deadline.startTimer("getPartitions"); List<Partition> partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); Deadline.stopTimer(); sharedCache.refreshPartitionsInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), partitions); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); } }
@Override public List<Partition> getPartitions(String catName, String dbName, String tblName, int max) throws MetaException, NoSuchObjectException { catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getPartitions(catName, dbName, tblName, max); } Table tbl = sharedCache.getTableFromCache(catName, dbName, tblName); if (tbl == null) { // The table containing the partitions is not yet loaded in cache return rawStore.getPartitions(catName, dbName, tblName, max); } List<Partition> parts = sharedCache.listCachedPartitions(catName, dbName, tblName, max); return parts; }
if (partNames == null) { if (0 != table.getPartitionKeysSize()) { for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) { alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition, validWriteIds, writeId);
if (table.isSetPartitionKeys()) { Deadline.startTimer("getPartitions"); partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); Deadline.stopTimer(); List<String> partNames = new ArrayList<>(partitions.size());
private List<Path> getLocationsForTruncate(final RawStore ms, final String catName, final String dbName, final String tableName, final Table table, final List<String> partNames) throws Exception { List<Path> locations = new ArrayList<>(); if (partNames == null) { if (0 != table.getPartitionKeysSize()) { for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) { locations.add(new Path(partition.getSd().getLocation())); } } else { locations.add(new Path(table.getSd().getLocation())); } } else { for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { locations.add(new Path(partition.getSd().getLocation())); } } return locations; }
@Override public List<Partition> get_partitions(final String db_name, final String tbl_name, final short max_parts) throws NoSuchObjectException, MetaException { String[] parsedDbName = parseDbName(db_name, conf); startTableFunction("get_partitions", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); List<Partition> ret = null; Exception ex = null; try { checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, NO_FILTER_STRING, max_parts); authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); ret = getMS().getPartitions(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, max_parts); ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (Exception e) { ex = e; throwMetaException(e); } finally { endFunction("get_partitions", ret != null, ex, tbl_name); } return ret; }
if (partNames == null) { if (0 != table.getPartitionKeysSize()) { for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) { alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition);
parts = msdb.getPartitions(catName, dbname, name, -1); Map<Partition, ColumnStatistics> columnStatsNeedUpdated = new HashMap<>(); for (Partition part : parts) { parts = msdb.getPartitions(catName, dbname, name, -1); for (Partition part : parts) { Partition oldPart = new Partition(part);
private static void dropAllStoreObjects(RawStore store) throws MetaException, InvalidObjectException, InvalidInputException { try { Deadline.registerIfNot(100000); Deadline.startTimer("getPartition"); List<String> dbs = store.getAllDatabases(DEFAULT_CATALOG_NAME); for (int i = 0; i < dbs.size(); i++) { String db = dbs.get(i); List<String> tbls = store.getAllTables(DEFAULT_CATALOG_NAME, db); for (String tbl : tbls) { List<Partition> parts = store.getPartitions(DEFAULT_CATALOG_NAME, db, tbl, 100); for (Partition part : parts) { store.dropPartition(DEFAULT_CATALOG_NAME, db, tbl, part.getValues()); } store.dropTable(DEFAULT_CATALOG_NAME, db, tbl); } store.dropDatabase(DEFAULT_CATALOG_NAME, db); } } catch (NoSuchObjectException e) { } }
for (String tbl : tbls) { Deadline.startTimer("getPartition"); List<Partition> parts = store.getPartitions(DEFAULT_CATALOG_NAME, db, tbl, 100); for (Partition part : parts) { store.dropPartition(DEFAULT_CATALOG_NAME, db, tbl, part.getValues());
@Override List<Partition> run(RawStore ms) throws Exception { return ms.getPartitions(db_name, tbl_name, max_parts); } });
@Override public List<Partition> getPartitions(String catName, String dbName, String tblName, int max) throws MetaException, NoSuchObjectException { catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { return rawStore.getPartitions(catName, dbName, tblName, max); } Table tbl = sharedCache.getTableFromCache(catName, dbName, tblName); if (tbl == null) { // The table containing the partitions is not yet loaded in cache return rawStore.getPartitions(catName, dbName, tblName, max); } List<Partition> parts = sharedCache.listCachedPartitions(catName, dbName, tblName, max); return parts; }
private void updateTablePartitions(RawStore rawStore, String catName, String dbName, String tblName) { try { Deadline.startTimer("getPartitions"); List<Partition> partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); Deadline.stopTimer(); sharedCache.refreshPartitionsInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), partitions); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); } }
public List<Partition> get_partitions(final String db_name, final String tbl_name, final short max_parts) throws NoSuchObjectException, MetaException { startTableFunction("get_partitions", db_name, tbl_name); List<Partition> ret = null; Exception ex = null; try { ret = getMS().getPartitions(db_name, tbl_name, max_parts); } catch (Exception e) { ex = e; if (e instanceof MetaException) { throw (MetaException) e; } else if (e instanceof NoSuchObjectException) { throw (NoSuchObjectException) e; } else { MetaException me = new MetaException(e.toString()); me.initCause(e); throw me; } } finally { endFunction("get_partitions", ret != null, ex, tbl_name); } return ret; }
@Override public List<Partition> get_partitions(final String db_name, final String tbl_name, final short max_parts) throws NoSuchObjectException, MetaException { startTableFunction("get_partitions", db_name, tbl_name); fireReadTablePreEvent(db_name, tbl_name); List<Partition> ret = null; Exception ex = null; try { ret = getMS().getPartitions(db_name, tbl_name, max_parts); } catch (Exception e) { ex = e; if (e instanceof MetaException) { throw (MetaException) e; } else if (e instanceof NoSuchObjectException) { throw (NoSuchObjectException) e; } else { throw newMetaException(e); } } finally { endFunction("get_partitions", ret != null, ex, tbl_name); } return ret; }
@Override public List<Partition> get_partitions(final String db_name, final String tbl_name, final short max_parts) throws NoSuchObjectException, MetaException { startTableFunction("get_partitions", db_name, tbl_name); fireReadTablePreEvent(db_name, tbl_name); List<Partition> ret = null; Exception ex = null; try { ret = getMS().getPartitions(db_name, tbl_name, max_parts); } catch (Exception e) { ex = e; if (e instanceof MetaException) { throw (MetaException) e; } else if (e instanceof NoSuchObjectException) { throw (NoSuchObjectException) e; } else { throw newMetaException(e); } } finally { endFunction("get_partitions", ret != null, ex, tbl_name); } return ret; }
List<Partition> partsToDelete = ms.getPartitions(dbName, tableName, partitionBatchSize); if (partsToDelete == null || partsToDelete.isEmpty()) { break;
List<Partition> partsToDelete = ms.getPartitions(dbName, tableName, partitionBatchSize); if (partsToDelete == null || partsToDelete.isEmpty()) { break;
@Override public List<Partition> get_partitions(final String db_name, final String tbl_name, final short max_parts) throws NoSuchObjectException, MetaException { String[] parsedDbName = parseDbName(db_name, conf); startTableFunction("get_partitions", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); List<Partition> ret = null; Exception ex = null; try { checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, NO_FILTER_STRING, max_parts); ret = getMS().getPartitions(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, max_parts); } catch (Exception e) { ex = e; throwMetaException(e); } finally { endFunction("get_partitions", ret != null, ex, tbl_name); } return ret; }