@Override public List<ColumnStatistics> getPartitionColumnStatistics( String catName, String dbName, String tblName, List<String> partNames, List<String> colNames, String writeIdList) throws MetaException, NoSuchObjectException { // TODO: why have updatePartitionColumnStatistics cache if this is a bypass? // Note: when implemented, this needs to call adjustColStatForGet, like other get methods. return rawStore.getPartitionColumnStatistics( catName, dbName, tblName, partNames, colNames, writeIdList); }
List<ColumnStatistics> list = getMS().getPartitionColumnStatistics( parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, Lists.newArrayList(convertedPartName), Lists.newArrayList(colName));
List<ColumnStatistics> stats = getMS().getPartitionColumnStatistics( catName, dbName, tblName, lowerCasePartNames, lowerCaseColNames, request.isSetValidWriteIdList() ? request.getValidWriteIdList() : null);
List<ColumnStatistics> csOlds = ms.getPartitionColumnStatistics(catName, dbName, tableName, partitionNames, colNames, request.getValidWriteIdList()); if (newStatsMap.values().size() != csOlds.size()) {
partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer();
List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(catName, dbname, tblname, oldPartNames, oldColNames); assert (partsColStats.size() <= 1);
private void updateTablePartitionColStats(RawStore rawStore, String catName, String dbName, String tblName) { boolean committed = false; rawStore.openTransaction(); try { Table table = rawStore.getTable(catName, dbName, tblName); List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); List<String> partNames = rawStore.listPartitionNames(catName, dbName, tblName, (short) -1); // Get partition column stats for this table Deadline.startTimer("getPartitionColumnStatistics"); List<ColumnStatistics> partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); List<Partition> parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); // Also save partitions for consistency as they have the stats state. for (Partition part : parts) { sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); } committed = rawStore.commitTransaction(); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); } finally { if (!committed) { sharedCache.removeAllPartitionColStatsFromCache(catName, dbName, tblName); rawStore.rollbackTransaction(); } } }
String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); List<ColumnStatistics> partColStatsList = getMS().getPartitionColumnStatistics(parsedCatName, parsedDbName, tblName, Collections.singletonList(partName), StatsSetupConst.getColumnsHavingStats(part.getParameters()));
@Override // TODO: calculate from cached values. public List<ColumnStatistics> getPartitionColumnStatistics(String catName, String dbName, String tblName, List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException { return rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); }
@Override public ColumnStatistics get_partition_column_statistics(String dbName, String tableName, String partName, String colName) throws NoSuchObjectException, MetaException, InvalidInputException, TException, InvalidObjectException { dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); String convertedPartName = lowerCaseConvertPartName(partName); startFunction("get_column_statistics_by_partition: db=" + dbName + " table=" + tableName + " partition=" + convertedPartName + " column=" + colName); ColumnStatistics statsObj = null; try { List<ColumnStatistics> list = getMS().getPartitionColumnStatistics(dbName, tableName, Lists.newArrayList(convertedPartName), Lists.newArrayList(colName)); if (list.isEmpty()) return null; if (list.size() != 1) { throw new MetaException(list.size() + " statistics for single column and partition"); } statsObj = list.get(0); } finally { endFunction("get_column_statistics_by_partition: ", statsObj != null, null, tableName); } return statsObj; }
@Override public ColumnStatistics get_partition_column_statistics(String dbName, String tableName, String partName, String colName) throws NoSuchObjectException, MetaException, InvalidInputException, TException, InvalidObjectException { dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); String convertedPartName = lowerCaseConvertPartName(partName); startFunction("get_column_statistics_by_partition: db=" + dbName + " table=" + tableName + " partition=" + convertedPartName + " column=" + colName); ColumnStatistics statsObj = null; try { List<ColumnStatistics> list = getMS().getPartitionColumnStatistics(dbName, tableName, Lists.newArrayList(convertedPartName), Lists.newArrayList(colName)); if (list.isEmpty()) return null; if (list.size() != 1) { throw new MetaException(list.size() + " statistics for single column and partition"); } statsObj = list.get(0); } finally { endFunction("get_column_statistics_by_partition: ", statsObj != null, null, tableName); } return statsObj; }
@Override public ColumnStatistics get_partition_column_statistics(String dbName, String tableName, String partName, String colName) throws TException { dbName = dbName.toLowerCase(); String[] parsedDbName = parseDbName(dbName, conf); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); String convertedPartName = lowerCaseConvertPartName(partName); startFunction("get_column_statistics_by_partition", ": table=" + getCatalogQualifiedTableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName) + " partition=" + convertedPartName + " column=" + colName); ColumnStatistics statsObj = null; try { List<ColumnStatistics> list = getMS().getPartitionColumnStatistics( parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, Lists.newArrayList(convertedPartName), Lists.newArrayList(colName)); if (list.isEmpty()) { return null; } if (list.size() != 1) { throw new MetaException(list.size() + " statistics for single column and partition"); } statsObj = list.get(0); } finally { endFunction("get_column_statistics_by_partition", statsObj != null, null, tableName); } return statsObj; }
public ColumnStatistics get_partition_column_statistics(String dbName, String tableName, String partName, String colName) throws NoSuchObjectException, MetaException, InvalidInputException, TException,InvalidObjectException { dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); colName = colName.toLowerCase(); String convertedPartName = lowerCaseConvertPartName(partName); startFunction("get_column_statistics_by_partition: db=" + dbName + " table=" + tableName + " partition=" + convertedPartName + " column=" + colName); ColumnStatistics statsObj = null; try { List<String> partVals = getPartValsFromName(getMS(), dbName, tableName, partName); statsObj = getMS().getPartitionColumnStatistics(dbName, tableName, convertedPartName, partVals, colName); } finally { endFunction("get_column_statistics_by_partition: ", statsObj != null, null, tableName); } return statsObj; }
@Override public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request) throws MetaException, NoSuchObjectException, TException { String dbName = request.getDbName().toLowerCase(); String tblName = request.getTblName().toLowerCase(); startFunction("get_partitions_statistics_req: db=" + dbName + " table=" + tblName); PartitionsStatsResult result = null; List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size()); for (String colName : request.getColNames()) { lowerCaseColNames.add(colName.toLowerCase()); } List<String> lowerCasePartNames = new ArrayList<String>(request.getPartNames().size()); for (String partName : request.getPartNames()) { lowerCasePartNames.add(lowerCaseConvertPartName(partName)); } try { List<ColumnStatistics> stats = getMS().getPartitionColumnStatistics( dbName, tblName, lowerCasePartNames, lowerCaseColNames); Map<String, List<ColumnStatisticsObj>> map = new HashMap<String, List<ColumnStatisticsObj>>(); for (ColumnStatistics stat : stats) { map.put(stat.getStatsDesc().getPartName(), stat.getStatsObj()); } result = new PartitionsStatsResult(map); } finally { endFunction("get_partitions_statistics_req: ", result == null, null, tblName); } return result; }
@Override public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request) throws MetaException, NoSuchObjectException, TException { String dbName = request.getDbName().toLowerCase(); String tblName = request.getTblName().toLowerCase(); startFunction("get_partitions_statistics_req: db=" + dbName + " table=" + tblName); PartitionsStatsResult result = null; List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size()); for (String colName : request.getColNames()) { lowerCaseColNames.add(colName.toLowerCase()); } List<String> lowerCasePartNames = new ArrayList<String>(request.getPartNames().size()); for (String partName : request.getPartNames()) { lowerCasePartNames.add(lowerCaseConvertPartName(partName)); } try { List<ColumnStatistics> stats = getMS().getPartitionColumnStatistics( dbName, tblName, lowerCasePartNames, lowerCaseColNames); Map<String, List<ColumnStatisticsObj>> map = new HashMap<String, List<ColumnStatisticsObj>>(); for (ColumnStatistics stat : stats) { map.put(stat.getStatsDesc().getPartName(), stat.getStatsObj()); } result = new PartitionsStatsResult(map); } finally { endFunction("get_partitions_statistics_req: ", result == null, null, tblName); } return result; }
private void updateTablePartitionColStats(RawStore rawStore, String catName, String dbName, String tblName) { try { Table table = rawStore.getTable(catName, dbName, tblName); List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); List<String> partNames = rawStore.listPartitionNames(catName, dbName, tblName, (short) -1); // Get partition column stats for this table Deadline.startTimer("getPartitionColumnStatistics"); List<ColumnStatistics> partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); } }
@Override public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request) throws TException { String catName = request.isSetCatName() ? request.getCatName().toLowerCase() : getDefaultCatalog(conf); String dbName = request.getDbName().toLowerCase(); String tblName = request.getTblName().toLowerCase(); startFunction("get_partitions_statistics_req", ": table=" + getCatalogQualifiedTableName(catName, dbName, tblName)); PartitionsStatsResult result = null; List<String> lowerCaseColNames = new ArrayList<>(request.getColNames().size()); for (String colName : request.getColNames()) { lowerCaseColNames.add(colName.toLowerCase()); } List<String> lowerCasePartNames = new ArrayList<>(request.getPartNames().size()); for (String partName : request.getPartNames()) { lowerCasePartNames.add(lowerCaseConvertPartName(partName)); } try { List<ColumnStatistics> stats = getMS().getPartitionColumnStatistics( catName, dbName, tblName, lowerCasePartNames, lowerCaseColNames); Map<String, List<ColumnStatisticsObj>> map = new HashMap<>(); for (ColumnStatistics stat : stats) { map.put(stat.getStatsDesc().getPartName(), stat.getStatsObj()); } result = new PartitionsStatsResult(map); } finally { endFunction("get_partitions_statistics_req", result == null, null, tblName); } return result; }
List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbName, tableName, oldPartNames, oldColNames); assert (partsColStats.size() <= 1);
List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbName, tableName, oldPartNames, oldColNames); assert (partsColStats.size() <= 1);
List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(catName, dbname, tblname, oldPartNames, oldColNames); assert (partsColStats.size() <= 1);