colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); stats.add(colStats);
private ColumnStatistics makeColumnStats( List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException { ColumnStatistics result = new ColumnStatistics(); result.setStatsDesc(csd); List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size()); for (Object[] row : list) { // LastAnalyzed is stored per column but thrift has it per several; // get the lowest for now as nobody actually uses this field. Object laObj = row[offset + 15]; if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > MetastoreDirectSqlUtils .extractSqlLong(laObj))) { csd.setLastAnalyzed(MetastoreDirectSqlUtils.extractSqlLong(laObj)); } csos.add(prepareCSObj(row, offset)); Deadline.checkTimeout(); } result.setStatsObj(csos); return result; }
private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException { // TODO: is this really needed? this code is propagated from HIVE-1362 but most of it is useless. ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); statsDesc.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName().toLowerCase() : getDefaultCatalog(conf)); statsDesc.setDbName(statsDesc.getDbName().toLowerCase()); statsDesc.setTableName(statsDesc.getTableName().toLowerCase()); statsDesc.setPartName(lowerCaseConvertPartName(statsDesc.getPartName())); long time = System.currentTimeMillis() / 1000; statsDesc.setLastAnalyzed(time); for (ColumnStatisticsObj statsObj : colStats.getStatsObj()) { statsObj.setColName(statsObj.getColName().toLowerCase()); statsObj.setColType(statsObj.getColType().toLowerCase()); } colStats.setStatsDesc(statsDesc); colStats.setStatsObj(colStats.getStatsObj()); }
public void setFieldValue(_Fields field, Object value) { switch (field) { case STATS_DESC: if (value == null) { unsetStatsDesc(); } else { setStatsDesc((ColumnStatisticsDesc)value); } break; case STATS_OBJ: if (value == null) { unsetStatsObj(); } else { setStatsObj((List<ColumnStatisticsObj>)value); } break; case IS_STATS_COMPLIANT: if (value == null) { unsetIsStatsCompliant(); } else { setIsStatsCompliant((Boolean)value); } break; } }
ColumnStatisticsDesc statsDesc = getColumnStatsDesc(names[0], names[1], partName, isTblLevel); ColumnStatistics colStats = new ColumnStatistics(); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); stats.add(colStats);
partName, partName == null); ColumnStatistics colStat = new ColumnStatistics(); colStat.setStatsDesc(statsDesc); colStat.addToStatsObj(statsObj); return colStat;
partName, partName == null); ColumnStatistics colStat = new ColumnStatistics(); colStat.setStatsDesc(statsDesc); colStat.addToStatsObj(statsObj); return colStat;
colStatObjs.add(colStats); stats.setStatsDesc(statsDesc); stats.setStatsObj(colStatObjs);
colStatObjs.add(colStats); stats.setStatsDesc(statsDesc); stats.setStatsObj(colStatObjs);
desc.setLastAnalyzed(now); desc.setPartName("ds=" + String.valueOf(i)); cs.setStatsDesc(desc); ColumnStatisticsObj obj = new ColumnStatisticsObj(); obj.setColName("col1");
colStatObjs.add(colStats); stats.setStatsDesc(statsDesc); stats.setStatsObj(colStatObjs);
colStatObjs.add(col3Stats); stats.setStatsDesc(statsDesc); stats.setStatsObj(colStatObjs);
colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); dbName, tblName, Lists.newArrayList(colName[1])).isEmpty()); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs);
desc.setTableName(tbl1.getTableName()); desc.setPartName("test_part_col=a" + i); stats.setStatsDesc(desc);
private ColumnStatistics convertToTableColumnStatistics(MTableColumnStatistics mStatsObj) throws MetaException { if (mStatsObj == null) { return null; } ColumnStatisticsDesc statsDesc = getTableColumnStatisticsDesc(mStatsObj); ColumnStatisticsObj statsObj = getTableColumnStatisticsObj(mStatsObj); List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>(); statsObjs.add(statsObj); ColumnStatistics colStats = new ColumnStatistics(); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); return colStats; }
private ColumnStatistics convertToPartColumnStatistics(MPartitionColumnStatistics mStatsObj) { if (mStatsObj == null) { return null; } ColumnStatisticsDesc statsDesc = getPartitionColumnStatisticsDesc(mStatsObj); ColumnStatisticsObj statsObj = getPartitionColumnStatisticsObj(mStatsObj); List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>(); statsObjs.add(statsObj); ColumnStatistics colStats = new ColumnStatistics(); colStats.setStatsDesc(statsDesc); colStats.setStatsObj(statsObjs); return colStats; }
public void setFieldValue(_Fields field, Object value) { switch (field) { case STATS_DESC: if (value == null) { unsetStatsDesc(); } else { setStatsDesc((ColumnStatisticsDesc)value); } break; case STATS_OBJ: if (value == null) { unsetStatsObj(); } else { setStatsObj((List<ColumnStatisticsObj>)value); } break; } }
private void setColumnStatistics(TableName tableName, TableStatistics tableStatistics, Table table, Predicate<FieldSchema> fieldsPredicate) throws TException { ColumnStatisticsDesc hiveColumnStatisticsDesc = new ColumnStatisticsDesc(); hiveColumnStatisticsDesc.setIsTblLevel(false); hiveColumnStatisticsDesc.setDbName(getSchema(tableName)); hiveColumnStatisticsDesc.setTableName(tableName.getSchemalessNameInDatabase()); List<ColumnStatisticsObj> hiveColumnStatisticsObjs = table.getSd().getCols().stream() .filter(fieldsPredicate) .map(fieldSchema -> toHiveColumnStatistics(fieldSchema, tableStatistics.getColumns().get(fieldSchema.getName()))) .collect(toList()); org.apache.hadoop.hive.metastore.api.ColumnStatistics hiveColumnStatistics = new org.apache.hadoop.hive.metastore.api.ColumnStatistics(); hiveColumnStatistics.setStatsDesc(hiveColumnStatisticsDesc); hiveColumnStatistics.setStatsObj(hiveColumnStatisticsObjs); client.update_table_column_statistics(hiveColumnStatistics); }
private ColumnStatistics makeColumnStats( List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException { ColumnStatistics result = new ColumnStatistics(); result.setStatsDesc(csd); List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size()); for (Object[] row : list) { // LastAnalyzed is stored per column but thrift has it per several; // get the lowest for now as nobody actually uses this field. Object laObj = row[offset + 14]; if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > extractSqlLong(laObj))) { csd.setLastAnalyzed(extractSqlLong(laObj)); } csos.add(prepareCSObj(row, offset)); Deadline.checkTimeout(); } result.setStatsObj(csos); return result; }
private void setColumnStatistics(TableName tableName, TableStatistics tableStatistics, Table table, Predicate<FieldSchema> fieldsPredicate) throws TException { ColumnStatisticsDesc hiveColumnStatisticsDesc = new ColumnStatisticsDesc(); hiveColumnStatisticsDesc.setIsTblLevel(false); hiveColumnStatisticsDesc.setDbName(getSchema(tableName)); hiveColumnStatisticsDesc.setTableName(tableName.getSchemalessNameInDatabase()); List<ColumnStatisticsObj> hiveColumnStatisticsObjs = table.getSd().getCols().stream() .filter(fieldsPredicate) .map(fieldSchema -> toHiveColumnStatistics(fieldSchema, tableStatistics.getColumns().get(fieldSchema.getName()))) .collect(toList()); org.apache.hadoop.hive.metastore.api.ColumnStatistics hiveColumnStatistics = new org.apache.hadoop.hive.metastore.api.ColumnStatistics(); hiveColumnStatistics.setStatsDesc(hiveColumnStatisticsDesc); hiveColumnStatistics.setStatsObj(hiveColumnStatisticsObjs); client.update_table_column_statistics(hiveColumnStatistics); }