private ColumnStatistics makeColumnStats( List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException { ColumnStatistics result = new ColumnStatistics(); result.setStatsDesc(csd); List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size()); for (Object[] row : list) { // LastAnalyzed is stored per column but thrift has it per several; // get the lowest for now as nobody actually uses this field. Object laObj = row[offset + 15]; if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > MetastoreDirectSqlUtils .extractSqlLong(laObj))) { csd.setLastAnalyzed(MetastoreDirectSqlUtils.extractSqlLong(laObj)); } csos.add(prepareCSObj(row, offset)); Deadline.checkTimeout(); } result.setStatsObj(csos); return result; }
private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException { // TODO: is this really needed? this code is propagated from HIVE-1362 but most of it is useless. ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); statsDesc.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf)); statsDesc.setDbName(statsDesc.getDbName().toLowerCase()); statsDesc.setTableName(statsDesc.getTableName().toLowerCase()); statsDesc.setPartName(lowerCaseConvertPartName(statsDesc.getPartName())); long time = System.currentTimeMillis() / 1000; statsDesc.setLastAnalyzed(time); for (ColumnStatisticsObj statsObj : colStats.getStatsObj()) { statsObj.setColName(statsObj.getColName().toLowerCase()); statsObj.setColType(statsObj.getColType().toLowerCase()); } colStats.setStatsDesc(statsDesc); colStats.setStatsObj(colStats.getStatsObj()); }
@Override public boolean equals(Object that) { if (that == null) return false; if (that instanceof ColumnStatistics) return this.equals((ColumnStatistics)that); return false; }
public Object getFieldValue(_Fields field) { switch (field) { case STATS_DESC: return getStatsDesc(); case STATS_OBJ: return getStatsObj(); case IS_STATS_COMPLIANT: return isIsStatsCompliant(); } throw new IllegalStateException(); }
public void setFieldValue(_Fields field, Object value) { switch (field) { case STATS_DESC: if (value == null) { unsetStatsDesc(); } else { setStatsDesc((ColumnStatisticsDesc)value); } break; case STATS_OBJ: if (value == null) { unsetStatsObj(); } else { setStatsObj((List<ColumnStatisticsObj>)value); } break; case IS_STATS_COMPLIANT: if (value == null) { unsetIsStatsCompliant(); } else { setIsStatsCompliant((Boolean)value); } break; } }
public static void getMergableCols(ColumnStatistics csNew, Map<String, String> parameters) { List<ColumnStatisticsObj> list = new ArrayList<>(); for (int index = 0; index < csNew.getStatsObj().size(); index++) { ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index); // canColumnStatsMerge guarantees that it is accurate before we do merge if (StatsSetupConst.canColumnStatsMerge(parameters, statsObjNew.getColName())) { list.add(statsObjNew); } // in all the other cases, we can not merge } csNew.setStatsObj(list); }
cachedStore.addPartition(ptn2); ColumnStatistics stats = new ColumnStatistics(); ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, dbName, tblName); statsDesc.setPartName("col"); colStatObjs.add(colStats); stats.setStatsDesc(statsDesc); stats.setStatsObj(colStatObjs); cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals1, null, -1); cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals2, null, -1);
new HashMap<String, ColumnStatsAggregator>(); for (ColumnStatistics css : partStats) { List<ColumnStatisticsObj> objs = css.getStatsObj(); for (ColumnStatisticsObj obj : objs) { String partName = css.getStatsDesc().getPartName(); if (aliasToAggregator.get(obj.getColName()) == null) { aliasToAggregator.put(obj.getColName(),
oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld); ColumnStatistics csOld = oldStatsMap.get(entry.getKey()); boolean isInvalidTxnStats = csOld != null && csOld.isSetIsStatsCompliant() && !csOld.isIsStatsCompliant(); Partition part = mapToPart.get(entry.getKey()); if (isInvalidTxnStats) { csNew.setStatsObj(Lists.newArrayList()); } else { if (csOld != null && csOld.getStatsObjSize() != 0 && !csNew.getStatsObj().isEmpty()) { MetaStoreServerUtils.mergeColStats(csNew, csOld); if (!csNew.getStatsObj().isEmpty()) { } else { LOG.debug("All the column stats " + csNew.getStatsDesc().getPartName() + " are not accurate to merge.");
ColumnStatisticsDesc statsDesc = getColumnStatsDesc(names[0], names[1], partName, partName == null); ColumnStatistics colStat = new ColumnStatistics(); colStat.setStatsDesc(statsDesc); colStat.addToStatsObj(statsObj); return colStat;
public void setFieldValue(_Fields field, Object value) { switch (field) { case STATS_DESC: if (value == null) { unsetStatsDesc(); } else { setStatsDesc((ColumnStatisticsDesc)value); } break; case STATS_OBJ: if (value == null) { unsetStatsObj(); } else { setStatsObj((List<ColumnStatisticsObj>)value); } break; } }
ColumnStatisticsDesc colStatsDesc = new ColumnStatisticsDesc(colStats.getStatsDesc()); colStatsDesc.setCatName(tbl.getCatName()); colStatsDesc.setDbName(getTableName()); colStatsDesc.setDbName(getDatabaseName()); tbl.getTTable().setColStats(new ColumnStatistics(colStatsDesc, colStats.getStatsObj()));
statsObjs.add(statsObj); ColumnStatistics colStats = new ColumnStatistics(); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); dbName, tblName, Lists.newArrayList(colName[1])).isEmpty()); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); statsDesc.setIsTblLevel(isTblLevel); colStats = new ColumnStatistics(); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); assertEquals(colStats.getStatsDesc().getPartName(), partName); assertEquals(colStats2.getColName(), colName[1]); assertEquals(colStats2.getStatsData().getStringStats().getMaxColLen(), maxColLen);
List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj(); for (ColumnStatisticsObj statsObj : statsObjs) { boolean found =false; if (updateColumnStats) { msdb.deletePartitionColumnStatistics(catName, dbname, tblname, partColStats.getStatsDesc().getPartName(), partVals, statsObj.getColName()); } else { deletedCols.add(statsObj.getColName()); msdb.deletePartitionColumnStatistics(catName, dbname, tblname, partColStats.getStatsDesc().getPartName(), partVals, statsObj.getColName()); partColStats.setStatsObj(newStatsObjs); newPartsColStats = partColStats;
for (int _i294 = 0; _i294 < _list292.size; ++_i294) _elem293 = new ColumnStatistics(); _elem293.read(iprot); struct.colStats.add(_elem293);
public ColumnStatistics deepCopy() { return new ColumnStatistics(this); }
ColumnStatistics colStats = work.getColStats(); LOG.debug("Got stats through replication for " + colStats.getStatsDesc().getDbName() + "." + colStats.getStatsDesc().getTableName()); return colStats; ColumnStatistics colStat = new ColumnStatistics(); colStat.setStatsDesc(statsDesc); colStat.addToStatsObj(statsObj); return colStat;
@Override public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws TException { String defaultCat = getDefaultCatalog(conf); for (ColumnStatistics stats : request.getColStats()) { if (!stats.getStatsDesc().isSetCatName()) { stats.getStatsDesc().setCatName(defaultCat); } } return client.set_aggr_stats_for(request); }
public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) throws InvalidObjectException { List<ColumnStatisticsObj> list = new ArrayList<>(); if (csNew.getStatsObj().size() != csOld.getStatsObjSize()) { csNew.getStatsObj().size(), csOld.getStatsObjSize()); for (ColumnStatisticsObj obj : csOld.getStatsObj()) { map.put(obj.getColName(), obj); for (int index = 0; index < csNew.getStatsObj().size(); index++) { ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index); ColumnStatisticsObj statsObjOld = map.get(statsObjNew.getColName()); if (statsObjOld != null) { csNew.setStatsObj(list);
private static void mergeColumnStats(Map<String, ColumnStatisticsObj> oldStats, ColumnStatistics newStats) { List<ColumnStatisticsObj> newColList = newStats.getStatsObj(); if (newColList != null) { for (ColumnStatisticsObj colStat : newColList) { // This is admittedly a bit simple, StatsObjectConverter seems to allow // old stats attributes to be kept if the new values do not overwrite them. oldStats.put(colStat.getColName().toLowerCase(), colStat); } } }