@Override List<Partition> getPartitionsByNames(CompactionInfo ci) throws MetaException { try { return rs.getPartitionsByNames(getDefaultCatalog(conf), ci.dbname, ci.tableName, Collections.singletonList(ci.partName)); } catch (MetaException e) { LOG.error("Unable to get partitions by name for CompactionInfo=" + ci); throw e; } catch (NoSuchObjectException e) { LOG.error("Unable to get partitions by name for CompactionInfo=" + ci); throw new MetaException(e.toString()); } } }
/** * Get the partition being compacted. * @param ci compaction info returned from the compaction queue * @return metastore partition, or null if there is not partition in this compaction info * @throws Exception if underlying calls throw, or if the partition name resolves to more than * one partition. */ protected Partition resolvePartition(CompactionInfo ci) throws Exception { if (ci.partName != null) { List<Partition> parts = null; try { parts = rs.getPartitionsByNames(ci.dbname, ci.tableName, Collections.singletonList(ci.partName)); if (parts == null || parts.size() == 0) { // The partition got dropped before we went looking for it. return null; } } catch (Exception e) { LOG.error("Unable to find partition " + ci.getFullPartitionName() + ", " + e.getMessage()); throw e; } if (parts.size() != 1) { LOG.error(ci.getFullPartitionName() + " does not refer to a single partition. " + parts); throw new MetaException("Too many partitions for : " + ci.getFullPartitionName()); } return parts.get(0); } else { return null; } }
private List<Path> getLocationsForTruncate(final RawStore ms, final String catName, final String dbName, final String tableName, final Table table, final List<String> partNames) throws Exception { List<Path> locations = new ArrayList<>(); if (partNames == null) { if (0 != table.getPartitionKeysSize()) { for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) { locations.add(new Path(partition.getSd().getLocation())); } } else { locations.add(new Path(table.getSd().getLocation())); } } else { for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { locations.add(new Path(partition.getSd().getLocation())); } } return locations; }
@Override public List<Partition> getPartitionsByNames(String catName, String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache return rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); } List<Partition> partitions = new ArrayList<>(); for (String partName : partNames) { Partition part = sharedCache.getPartitionFromCache(catName, dbName, tblName, partNameToVals(partName)); if (part!=null) { partitions.add(part); } } return partitions; }
nextBatchStart = nextBatchEnd; try { currentBatch = rs.getPartitionsByNames(cat, db, tbl, currentNames); } catch (NoSuchObjectException e) { LOG.error("Failed to get partitions for " + fullTableName + ", skipping some partitions", e);
List<Partition> partitions = ms.getPartitionsByNames(catName, dbName, tableName, partitionNames); Map<String, Partition> mapToPart = new HashMap<>(); for (int index = 0; index < partitionNames.size(); index++) {
for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition, validWriteIds, writeId);
partNames = spec.getNames(); minCount = partNames.size(); parts = ms.getPartitionsByNames(catName, dbName, tblName, partNames); } else { throw new MetaException("Partition spec is not set");
private void updateTablePartitionColStats(RawStore rawStore, String catName, String dbName, String tblName) { boolean committed = false; rawStore.openTransaction(); try { Table table = rawStore.getTable(catName, dbName, tblName); List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); List<String> partNames = rawStore.listPartitionNames(catName, dbName, tblName, (short) -1); // Get partition column stats for this table Deadline.startTimer("getPartitionColumnStatistics"); List<ColumnStatistics> partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); List<Partition> parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); // Also save partitions for consistency as they have the stats state. for (Partition part : parts) { sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); } committed = rawStore.commitTransaction(); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); } finally { if (!committed) { sharedCache.removeAllPartitionColStatsFromCache(catName, dbName, tblName); rawStore.rollbackTransaction(); } } }
ret = getMS().getPartitionsByNames(parsedCatName, parsedDbName, tblName, partNames); ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
List<String> nameBatch = partNames.subList(index, index + currentBatchSize); index += currentBatchSize; List<Partition> parts = ms.getPartitionsByNames(DEFAULT_CATALOG_NAME, dbName, tblName, nameBatch); for (Partition part : parts) { if (!part.isSetSd() || !part.getSd().isSetLocation()) {
/** * Get the partition being compacted. * @param ci compaction info returned from the compaction queue * @return metastore partition, or null if there is not partition in this compaction info * @throws Exception if underlying calls throw, or if the partition name resolves to more than * one partition. */ protected Partition resolvePartition(CompactionInfo ci) throws Exception { if (ci.partName != null) { List<Partition> parts = null; try { parts = rs.getPartitionsByNames(ci.dbname, ci.tableName, Collections.singletonList(ci.partName)); if (parts == null || parts.size() == 0) { // The partition got dropped before we went looking for it. return null; } } catch (Exception e) { LOG.error("Unable to find partition " + ci.getFullPartitionName() + ", " + e.getMessage()); throw e; } if (parts.size() != 1) { LOG.error(ci.getFullPartitionName() + " does not refer to a single partition"); throw new MetaException("Too many partitions"); } return parts.get(0); } else { return null; } }
private List<Path> getLocationsForTruncate(final RawStore ms, final String catName, final String dbName, final String tableName, final Table table, final List<String> partNames) throws Exception { List<Path> locations = new ArrayList<>(); if (partNames == null) { if (0 != table.getPartitionKeysSize()) { for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) { locations.add(new Path(partition.getSd().getLocation())); } } else { locations.add(new Path(table.getSd().getLocation())); } } else { for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { locations.add(new Path(partition.getSd().getLocation())); } } return locations; }
@Override public List<Partition> getPartitionsByNames(String catName, String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { return rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache return rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); } List<Partition> partitions = new ArrayList<>(); for (String partName : partNames) { Partition part = sharedCache.getPartitionFromCache(catName, dbName, tblName, partNameToVals(partName)); if (part!=null) { partitions.add(part); } } return partitions; }
@Override public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> partNames) throws MetaException, NoSuchObjectException, TException { startTableFunction("get_partitions_by_names", dbName, tblName); List<Partition> ret = null; Exception ex = null; try { ret = getMS().getPartitionsByNames(dbName, tblName, partNames); } catch (Exception e) { ex = e; if (e instanceof MetaException) { throw (MetaException) e; } else if (e instanceof NoSuchObjectException) { throw (NoSuchObjectException) e; } else if (e instanceof TException) { throw (TException) e; } else { MetaException me = new MetaException(e.toString()); me.initCause(e); throw me; } } finally { endFunction("get_partitions_by_names", ret != null, ex, tblName); } return ret; }
@Override public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> partNames) throws MetaException, NoSuchObjectException, TException { startTableFunction("get_partitions_by_names", dbName, tblName); fireReadTablePreEvent(dbName, tblName); List<Partition> ret = null; Exception ex = null; try { ret = getMS().getPartitionsByNames(dbName, tblName, partNames); } catch (Exception e) { ex = e; rethrowException(e); } finally { endFunction("get_partitions_by_names", ret != null, ex, tblName); } return ret; }
@Override public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> partNames) throws MetaException, NoSuchObjectException, TException { startTableFunction("get_partitions_by_names", dbName, tblName); fireReadTablePreEvent(dbName, tblName); List<Partition> ret = null; Exception ex = null; try { ret = getMS().getPartitionsByNames(dbName, tblName, partNames); } catch (Exception e) { ex = e; rethrowException(e); } finally { endFunction("get_partitions_by_names", ret != null, ex, tblName); } return ret; }
@Override public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> partNames) throws TException { String[] parsedDbName = parseDbName(dbName, conf); startTableFunction("get_partitions_by_names", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName); fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName); List<Partition> ret = null; Exception ex = null; try { ret = getMS().getPartitionsByNames(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName, partNames); } catch (Exception e) { ex = e; rethrowException(e); } finally { endFunction("get_partitions_by_names", ret != null, ex, tblName); } return ret; }
for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { alterPartitionForTruncate(ms, catName, dbName, tableName, table, partition);
List<String> nameBatch = partNames.subList(index, index + currentBatchSize); index += currentBatchSize; List<Partition> parts = ms.getPartitionsByNames(DEFAULT_CATALOG_NAME, dbName, tblName, nameBatch); for (Partition part : parts) { if (!part.isSetSd() || !part.getSd().isSetLocation()) {