public Object getFieldValue(_Fields field) { switch (field) { case STATS_DESC: return getStatsDesc(); case STATS_OBJ: return getStatsObj(); case IS_STATS_COMPLIANT: return isIsStatsCompliant(); } throw new IllegalStateException(); }
@Override public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws TException { String defaultCat = getDefaultCatalog(conf); for (ColumnStatistics stats : request.getColStats()) { if (!stats.getStatsDesc().isSetCatName()) { stats.getStatsDesc().setCatName(defaultCat); } } return client.set_aggr_stats_for(request); }
public JSONUpdateTableColumnStatMessage(String server, String servicePrincipal, Long timestamp, ColumnStatistics colStats, Table tableObj, Map<String, String> parameters, String validWriteIds, long writeId) { this.timestamp = timestamp; this.server = server; this.servicePrincipal = servicePrincipal; this.writeId = writeId; this.validWriteIds = validWriteIds; this.database = colStats.getStatsDesc().getDbName(); try { this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); this.tableObjJson = MessageBuilder.createTableObjJson(tableObj); } catch (TException e) { throw new IllegalArgumentException("Could not serialize JSONUpdateTableColumnStatMessage : ", e); } this.parameters = parameters; }
public JSONUpdatePartitionColumnStatMessage(String server, String servicePrincipal, Long timestamp, ColumnStatistics colStats, List<String> partVals, Map<String, String> parameters, Table tableObj, String validWriteIds, long writeId) { this.timestamp = timestamp; this.server = server; this.servicePrincipal = servicePrincipal; this.writeId = writeId; this.validWriteIds = validWriteIds; this.database = colStats.getStatsDesc().getDbName(); this.partVals = partVals; try { this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); this.tableObjJson = MessageBuilder.createTableObjJson(tableObj); } catch (TException e) { throw new IllegalArgumentException("Could not serialize JSONUpdatePartitionColumnStatMessage : ", e); } this.parameters = parameters; }
/** {@inheritDoc} */ @Override public boolean updateTableColumnStatistics(ColumnStatistics statsObj) throws TException { if (!statsObj.getStatsDesc().isSetCatName()) { statsObj.getStatsDesc().setCatName(getDefaultCatalog(conf)); } // Note: currently this method doesn't set txn properties and thus won't work on txn tables. SetPartitionsStatsRequest req = new SetPartitionsStatsRequest(); req.addToColStats(statsObj); req.setNeedMerge(false); return client.update_table_column_statistics_req(req).isResult(); }
@Override public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj) throws TException { if (!statsObj.getStatsDesc().isSetCatName()) { statsObj.getStatsDesc().setCatName(getDefaultCatalog(conf)); } // Note: currently this method doesn't set txn properties and thus won't work on txn tables. SetPartitionsStatsRequest req = new SetPartitionsStatsRequest(); req.addToColStats(statsObj); req.setNeedMerge(false); return client.update_partition_column_statistics_req(req).isResult(); }
/** {@inheritDoc} */ @Override public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws NoSuchObjectException, InvalidObjectException, MetaException, TException, InvalidInputException { if (request.getColStatsSize() == 1) { ColumnStatistics colStats = request.getColStatsIterator().next(); ColumnStatisticsDesc desc = colStats.getStatsDesc(); String dbName = desc.getDbName().toLowerCase(); String tableName = desc.getTableName().toLowerCase(); if (getTempTable(dbName, tableName) != null) { return updateTempTableColumnStats(dbName, tableName, colStats); } } return super.setPartitionColumnStatistics(request); }
List<ColumnStatisticsObj> objs = css.getStatsObj(); for (ColumnStatisticsObj obj : objs) { String partName = css.getStatsDesc().getPartName(); if (aliasToAggregator.get(obj.getColName()) == null) { aliasToAggregator.put(obj.getColName(),
/** {@inheritDoc} */ @Override public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws NoSuchObjectException, InvalidObjectException, MetaException, TException, InvalidInputException { if (request.getColStatsSize() == 1) { ColumnStatistics colStats = request.getColStatsIterator().next(); ColumnStatisticsDesc desc = colStats.getStatsDesc(); String dbName = desc.getDbName().toLowerCase(); String tableName = desc.getTableName().toLowerCase(); if (getTempTable(dbName, tableName) != null) { return updateTempTableColumnStats(dbName, tableName, colStats); } } return super.setPartitionColumnStatistics(request); }
@Override public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats, List<String> partVals, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { Map<String, String> newParams = rawStore.updatePartitionColumnStatistics( colStats, partVals, validWriteIds, writeId); // in case of event based cache update, cache is updated during commit txn if (newParams != null && !canUseEvents) { String catName = colStats.getStatsDesc().isSetCatName() ? normalizeIdentifier(colStats.getStatsDesc().getCatName()) : DEFAULT_CATALOG_NAME; String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName()); String tblName = normalizeIdentifier(colStats.getStatsDesc().getTableName()); if (!shouldCacheTable(catName, dbName, tblName)) { return newParams; } Partition part = getPartition(catName, dbName, tblName, partVals); part.setParameters(newParams); sharedCache.alterPartitionInCache(catName, dbName, tblName, partVals, part); sharedCache.updatePartitionColStatsInCache(catName, dbName, tblName, partVals, colStats.getStatsObj()); } return newParams; }
partVal = Warehouse.makeValsFromName(cs.getStatsDesc().getPartName(), null); List<ColumnStatisticsObj> colStatsObjs = cs.getStatsObj(); for (ColumnStatisticsObj colStatObj : colStatsObjs) {
private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException { // TODO: is this really needed? this code is propagated from HIVE-1362 but most of it is useless. ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); statsDesc.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf)); statsDesc.setDbName(statsDesc.getDbName().toLowerCase()); statsDesc.setTableName(statsDesc.getTableName().toLowerCase()); statsDesc.setPartName(lowerCaseConvertPartName(statsDesc.getPartName())); long time = System.currentTimeMillis() / 1000; statsDesc.setLastAnalyzed(time); for (ColumnStatisticsObj statsObj : colStats.getStatsObj()) { statsObj.setColName(statsObj.getColName().toLowerCase()); statsObj.setColType(statsObj.getColType().toLowerCase()); } colStats.setStatsDesc(statsDesc); colStats.setStatsObj(colStats.getStatsObj()); }
map.put(stat.getStatsDesc().getPartName(), stat.getStatsObj());
@Override public List<Task<? extends Serializable>> handle(Context context) throws SemanticException { UpdatePartitionColumnStatMessage upcsm = deserializer.getUpdatePartitionColumnStatMessage(context.dmd.getPayload()); // Update tablename and database name in the statistics object ColumnStatistics colStats = upcsm.getColumnStatistics(); ColumnStatisticsDesc colStatsDesc = colStats.getStatsDesc(); if (!context.isTableNameEmpty()) { colStatsDesc.setTableName(context.tableName); } if (!context.isDbNameEmpty()) { colStatsDesc.setDbName(context.dbName); updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, context.tableName, null); } return Collections.singletonList(TaskFactory.get(new ColumnStatsUpdateWork(colStats), context.hiveConf)); } }
public boolean setPartitionColumnStatistics( SetPartitionsStatsRequest request) throws HiveException { try { ColumnStatistics colStat = request.getColStats().get(0); ColumnStatisticsDesc statsDesc = colStat.getStatsDesc(); Table tbl = getTable(statsDesc.getDbName(), statsDesc.getTableName()); AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, true); request.setValidWriteIdList(tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); request.setWriteId(tableSnapshot != null ? tableSnapshot.getWriteId() : 0); return getMSC().setPartitionColumnStatistics(request); } catch (Exception e) { LOG.debug(StringUtils.stringifyException(e)); throw new HiveException(e); } }
private int persistColumnStats(Hive db) throws HiveException, MetaException, IOException { ColumnStatistics colStats = constructColumnStatsFromInput(); ColumnStatisticsDesc colStatsDesc = colStats.getStatsDesc(); // We do not support stats replication for a transactional table yet. If we are converting // a non-transactional table to a transactional table during replication, we might get // column statistics but we shouldn't update those. if (work.getColStats() != null && AcidUtils.isTransactionalTable(getHive().getTable(colStatsDesc.getDbName(), colStatsDesc.getTableName()))) { LOG.debug("Skipped updating column stats for table " + TableName.getDbTable(colStatsDesc.getDbName(), colStatsDesc.getTableName()) + " because it is converted to a transactional table during replication."); return 0; } SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(Collections.singletonList(colStats)); db.setPartitionColumnStatistics(request); return 0; }
@Override public List<Task<? extends Serializable>> handle(Context context) throws SemanticException { UpdateTableColumnStatMessage utcsm = deserializer.getUpdateTableColumnStatMessage(context.dmd.getPayload()); // Update tablename and database name in the statistics object ColumnStatistics colStats = utcsm.getColumnStatistics(); ColumnStatisticsDesc colStatsDesc = colStats.getStatsDesc(); colStatsDesc.setDbName(context.dbName); if (!context.isTableNameEmpty()) { colStatsDesc.setTableName(context.tableName); } if (!context.isDbNameEmpty()) { updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, context.tableName, null); } // TODO: For txn stats update, ColumnStatsUpdateTask.execute()->Hive // .setPartitionColumnStatistics expects a valid writeId allocated by the current txn and // also, there should be a table snapshot. But, it won't be there as update from // ReplLoadTask which doesn't have a write id allocated. Need to check this further. return Collections.singletonList(TaskFactory.get(new ColumnStatsUpdateWork(colStats), context.hiveConf)); } }
Warehouse.getPartValuesFromPartName(cs.getStatsDesc().getPartName())); if (mpart == null || !isCurrentStatsValidForTheQuery(mpart, writeIdList, false)) {
@Override public void onUpdateTableColumnStat(UpdateTableColumnStatEvent updateTableColumnStatEvent) throws MetaException { UpdateTableColumnStatMessage msg = MessageBuilder.getInstance() .buildUpdateTableColumnStatMessage(updateTableColumnStatEvent.getColStats(), updateTableColumnStatEvent.getTableObj(), updateTableColumnStatEvent.getTableParameters(), updateTableColumnStatEvent.getValidWriteIds(), updateTableColumnStatEvent.getWriteId()); NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_TABLE_COLUMN_STAT.toString(), msgEncoder.getSerializer().serialize(msg)); ColumnStatisticsDesc statDesc = updateTableColumnStatEvent.getColStats().getStatsDesc(); event.setCatName(statDesc.isSetCatName() ? statDesc.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(statDesc.getDbName()); event.setTableName(statDesc.getTableName()); process(event, updateTableColumnStatEvent); }
@Override public void onUpdatePartitionColumnStat(UpdatePartitionColumnStatEvent updatePartColStatEvent) throws MetaException { UpdatePartitionColumnStatMessage msg = MessageBuilder.getInstance() .buildUpdatePartitionColumnStatMessage(updatePartColStatEvent.getPartColStats(), updatePartColStatEvent.getPartVals(), updatePartColStatEvent.getPartParameters(), updatePartColStatEvent.getTableObj(), updatePartColStatEvent.getValidWriteIds(), updatePartColStatEvent.getWriteId()); NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_PARTITION_COLUMN_STAT.toString(), msgEncoder.getSerializer().serialize(msg)); ColumnStatisticsDesc statDesc = updatePartColStatEvent.getPartColStats().getStatsDesc(); event.setCatName(statDesc.isSetCatName() ? statDesc.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(statDesc.getDbName()); event.setTableName(statDesc.getTableName()); process(event, updatePartColStatEvent); }