private long partsFoundForPartitions(String dbName, String tableName, List<String> partNames, List<String> colNames) throws MetaException { assert !colNames.isEmpty() && !partNames.isEmpty(); long partsFound = 0; boolean doTrace = LOG.isDebugEnabled(); String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " group by \"PARTITION_NAME\""; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); Object qResult = executeWithArray(query, prepareParams( dbName, tableName, partNames, colNames), queryText); long end = doTrace ? System.nanoTime() : 0; timingTrace(doTrace, queryText, start, end); ForwardQueryResult fqr = (ForwardQueryResult) qResult; Iterator<?> iter = fqr.iterator(); while (iter.hasNext()) { if (extractSqlLong(iter.next()) == colNames.size()) { partsFound++; } } return partsFound; }
@Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) { SkewedInfo skewedInfo = new SkewedInfo(); skewedInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>()); t.setSkewedInfo(skewedInfo); } Map<List<String>, String> skewMap = t.getSkewedInfo().getSkewedColValueLocationMaps(); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = new ArrayList<String>(); // left outer join produced a list with no values currentListId = null; } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; } else { skewMap.remove(currentList); // value based compare.. remove first } currentList.add((String)fields[3]); } skewMap.put(currentList, (String)fields[2]); }}); } // if (hasSkewedColumns)
private long partsFoundForPartitions(String dbName, String tableName, List<String> partNames, List<String> colNames) throws MetaException { assert !colNames.isEmpty() && !partNames.isEmpty(); long partsFound = 0; boolean doTrace = LOG.isDebugEnabled(); String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " group by \"PARTITION_NAME\""; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); Object qResult = executeWithArray(query, prepareParams( dbName, tableName, partNames, colNames), queryText); long end = doTrace ? System.nanoTime() : 0; timingTrace(doTrace, queryText, start, end); ForwardQueryResult fqr = (ForwardQueryResult) qResult; Iterator<?> iter = fqr.iterator(); while (iter.hasNext()) { if (extractSqlLong(iter.next()) == colNames.size()) { partsFound++; } } return partsFound; }
@Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) { SkewedInfo skewedInfo = new SkewedInfo(); skewedInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>()); t.setSkewedInfo(skewedInfo); } Map<List<String>, String> skewMap = t.getSkewedInfo().getSkewedColValueLocationMaps(); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = new ArrayList<String>(); // left outer join produced a list with no values currentListId = null; } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; } else { skewMap.remove(currentList); // value based compare.. remove first } currentList.add((String)fields[3]); } skewMap.put(currentList, (String)fields[2]); }}); } // if (hasSkewedColumns)
private ColumnStatistics makeColumnStats( List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException { ColumnStatistics result = new ColumnStatistics(); result.setStatsDesc(csd); List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size()); for (Object[] row : list) { // LastAnalyzed is stored per column but thrift has it per several; // get the lowest for now as nobody actually uses this field. Object laObj = row[offset + 14]; if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > extractSqlLong(laObj))) { csd.setLastAnalyzed(extractSqlLong(laObj)); } csos.add(prepareCSObj(row, offset)); Deadline.checkTimeout(); } result.setStatsObj(csos); return result; }
private ColumnStatistics makeColumnStats( List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException { ColumnStatistics result = new ColumnStatistics(); result.setStatsDesc(csd); List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size()); for (Object[] row : list) { // LastAnalyzed is stored per column but thrift has it per several; // get the lowest for now as nobody actually uses this field. Object laObj = row[offset + 14]; if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > extractSqlLong(laObj))) { csd.setLastAnalyzed(extractSqlLong(laObj)); } csos.add(prepareCSObj(row, offset)); Deadline.checkTimeout(); } result.setStatsObj(csos); return result; }
fields = iter.next(); long nestedId = extractSqlLong(fields[keyIndex]); if (nestedId < id) throw new MetaException("Found entries for unknown ID " + nestedId); if (nestedId > id) break; // fields belong to one of the next entries
if (colType.equals("boolean")) { BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); boolStats.setNumFalses(MetaStoreDirectSql.extractSqlLong(falses)); boolStats.setNumTrues(MetaStoreDirectSql.extractSqlLong(trues)); boolStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); data.setBooleanStats(boolStats); } else if (colType.equals("string") || colType.startsWith("varchar") || colType.startsWith("char")) { StringColumnStatsData stringStats = new StringColumnStatsData(); stringStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); stringStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); stringStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); stringStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); data.setStringStats(stringStats); } else if (colType.equals("binary")) { BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); binaryStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); binaryStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); binaryStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); data.setBinaryStats(binaryStats); } else if (colType.equals("bigint") || colType.equals("int") || colType.equals("timestamp")) { LongColumnStatsData longStats = new LongColumnStatsData(); longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); if (lhigh != null) { longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh)); longStats.setLowValue(MetaStoreDirectSql.extractSqlLong(llow));
@Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = null; // left outer join produced a list with no values currentListId = null; t.getSkewedInfo().addToSkewedColValues(new ArrayList<String>()); } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; t.getSkewedInfo().addToSkewedColValues(currentList); } currentList.add((String)fields[2]); } }});
@Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = null; // left outer join produced a list with no values currentListId = null; t.getSkewedInfo().addToSkewedColValues(new ArrayList<String>()); } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; t.getSkewedInfo().addToSkewedColValues(currentList); } currentList.add((String)fields[2]); } }});
if (colType.equals("boolean")) { BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); boolStats.setNumFalses(MetaStoreDirectSql.extractSqlLong(falses)); boolStats.setNumTrues(MetaStoreDirectSql.extractSqlLong(trues)); boolStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); data.setBooleanStats(boolStats); } else if (colType.equals("string") || colType.startsWith("varchar") || colType.startsWith("char")) { StringColumnStatsData stringStats = new StringColumnStatsData(); stringStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); stringStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); stringStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); stringStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); data.setStringStats(stringStats); } else if (colType.equals("binary")) { BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); binaryStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); binaryStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); binaryStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); data.setBinaryStats(binaryStats); } else if (colType.equals("bigint") || colType.equals("int") || colType.equals("timestamp")) { LongColumnStatsData longStats = new LongColumnStatsData(); longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); if (lhigh != null) { longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh)); longStats.setLowValue(MetaStoreDirectSql.extractSqlLong(llow));
if (colType.equals("boolean")) { BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); boolStats.setNumFalses(MetaStoreDirectSql.extractSqlLong(falses)); boolStats.setNumTrues(MetaStoreDirectSql.extractSqlLong(trues)); boolStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); data.setBooleanStats(boolStats); } else if (colType.equals("string") || colType.startsWith("varchar") || colType.startsWith("char")) { StringColumnStatsData stringStats = new StringColumnStatsData(); stringStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); stringStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); stringStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); stringStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); data.setStringStats(stringStats); } else if (colType.equals("binary")) { BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); binaryStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); binaryStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); binaryStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); data.setBinaryStats(binaryStats); } else if (colType.equals("bigint") || colType.equals("int") || colType.equals("smallint") || colType.equals("tinyint") || colType.equals("timestamp")) { LongColumnStatsData longStats = new LongColumnStatsData(); longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); if (lhigh != null) { longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh)); longStats.setLowValue(MetaStoreDirectSql.extractSqlLong(llow));
fields = iter.next(); long nestedId = extractSqlLong(fields[keyIndex]); if (nestedId < id) throw new MetaException("Found entries for unknown ID " + nestedId); if (nestedId > id) break; // fields belong to one of the next entries
if (colType.equals("boolean")) { BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); boolStats.setNumFalses(MetaStoreDirectSql.extractSqlLong(falses)); boolStats.setNumTrues(MetaStoreDirectSql.extractSqlLong(trues)); boolStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); data.setBooleanStats(boolStats); } else if (colType.equals("string") || colType.startsWith("varchar") || colType.startsWith("char")) { StringColumnStatsData stringStats = new StringColumnStatsData(); stringStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); stringStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); stringStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); stringStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); data.setStringStats(stringStats); } else if (colType.equals("binary")) { BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); binaryStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); binaryStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); binaryStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); data.setBinaryStats(binaryStats); } else if (colType.equals("bigint") || colType.equals("int") || colType.equals("smallint") || colType.equals("tinyint") || colType.equals("timestamp")) { LongColumnStatsData longStats = new LongColumnStatsData(); longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); if (lhigh != null) { longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh)); longStats.setLowValue(MetaStoreDirectSql.extractSqlLong(llow));
Long count = extractSqlLong(row[2]); if (count == partNames.size() || count < 2) { noExtraColumnNames.add(colName); row[2 + colStatIndex] = null; } else { Long val = extractSqlLong(o); row[2 + colStatIndex] = (Long) (val / sumVal * (partNames.size()));
partSb.append(extractSqlLong(partitionId)).append(","); for (Object[] fields : sqlResult) { long partitionId = extractSqlLong(fields[0]); Long sdId = extractSqlLong(fields[1]); Long colId = extractSqlLong(fields[2]); Long serdeId = extractSqlLong(fields[3]);
Long dbid = extractSqlLong(dbline[0]);
partSb.append(extractSqlLong(partitionId)).append(","); for (Object[] fields : sqlResult) { long partitionId = extractSqlLong(fields[0]); Long sdId = extractSqlLong(fields[1]); Long colId = extractSqlLong(fields[2]); Long serdeId = extractSqlLong(fields[3]);
Long count = extractSqlLong(row[2]); if (count == partNames.size() || count < 2) { noExtraColumnNames.add(colName); row[2 + colStatIndex] = null; } else { Long val = extractSqlLong(o); row[2 + colStatIndex] = (Long) (val / sumVal * (partNames.size()));
Long dbid = extractSqlLong(dbline[0]);