private void checkColTypeChangeCompatible(List<FieldSchema> oldCols, List<FieldSchema> newCols) throws InvalidOperationException { List<String> incompatibleCols = new ArrayList<>(); int maxCols = Math.min(oldCols.size(), newCols.size()); for (int i = 0; i < maxCols; i++) { if (!ColumnType.areColTypesCompatible( ColumnType.getTypeName(oldCols.get(i).getType()), ColumnType.getTypeName(newCols.get(i).getType()))) { incompatibleCols.add(newCols.get(i).getName()); } } if (!incompatibleCols.isEmpty()) { throw new InvalidOperationException( "The following columns have types incompatible with the existing " + "columns in their respective positions :\n" + org.apache.commons.lang.StringUtils.join(incompatibleCols, ',') ); } }
public static boolean columnsIncludedByNameType(List<FieldSchema> oldCols, List<FieldSchema> newCols) { if (oldCols.size() > newCols.size()) { return false; } Map<String, String> columnNameTypePairMap = new HashMap<>(newCols.size()); for (FieldSchema newCol : newCols) { columnNameTypePairMap.put(newCol.getName().toLowerCase(), newCol.getType()); } for (final FieldSchema oldCol : oldCols) { if (!columnNameTypePairMap.containsKey(oldCol.getName()) || !columnNameTypePairMap.get(oldCol.getName()).equalsIgnoreCase(oldCol.getType())) { return false; } } return true; }
/** * Generate normalized name for a given target column. */ private static String getTargetFieldName(int fieldIndex, String destTableName, List<String> colNames, List<FieldSchema> fieldSchemas) { String fieldName = fieldSchemas.get(fieldIndex).getName(); String[] parts = fieldName.split("\\."); if (destTableName != null) { String colName = parts[parts.length - 1]; if (colNames != null && !colNames.contains(colName)) { colName = colNames.get(fieldIndex); } return destTableName + "." + colName; } if (parts.length == 2 && parts[0].startsWith("_u")) { return parts[1]; } return fieldName; }
public static String getColumnNameDelimiter(List<FieldSchema> fieldSchemas) { // we first take a look if any fieldSchemas contain COMMA for (int i = 0; i < fieldSchemas.size(); i++) { if (fieldSchemas.get(i).getName().contains(",")) { return String.valueOf(ColumnType.COLUMN_COMMENTS_DELIMITER); } } return String.valueOf(','); }
/** * Parse the {@link #REPLACED_PARTITIONS_HIVE_METASTORE_KEY} from partition parameters to returns DDLs for all the partitions to be * dropped. * * @return A {@link List} of partitions to be dropped. Each element of the list is a {@link Map} which maps a partition's * key and value. * */ public static List<Map<String, String>> getDropPartitionsDDLInfo(Partition hivePartition) { List<Map<String, String>> replacedPartitionsDDLInfo = Lists.newArrayList(); List<FieldSchema> partitionKeys = hivePartition.getTable().getPartitionKeys(); if (StringUtils.isNotBlank(hivePartition.getParameters().get(REPLACED_PARTITIONS_HIVE_METASTORE_KEY))) { // Partitions are separated by "|" for (String partitionsInfoString : Splitter.on("|").omitEmptyStrings().split(hivePartition.getParameters().get(REPLACED_PARTITIONS_HIVE_METASTORE_KEY))) { // Values for a partition are separated by "," List<String> partitionValues = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(partitionsInfoString); // Do not drop the partition being processed. Sometimes a partition may have replaced another partition of the same values. if (!partitionValues.equals(hivePartition.getValues())) { ImmutableMap.Builder<String, String> partitionDDLInfoMap = ImmutableMap.builder(); for (int i = 0; i < partitionKeys.size(); i++) { partitionDDLInfoMap.put(partitionKeys.get(i).getName(), partitionValues.get(i)); } replacedPartitionsDDLInfo.add(partitionDDLInfoMap.build()); } } } return replacedPartitionsDDLInfo; }
private boolean checkPartialPartKeysEqual(List<FieldSchema> oldPartKeys, List<FieldSchema> newPartKeys) { //return true if both are null, or false if one is null and the other isn't if (newPartKeys == null || oldPartKeys == null) { return oldPartKeys == newPartKeys; } if (oldPartKeys.size() != newPartKeys.size()) { return false; } Iterator<FieldSchema> oldPartKeysIter = oldPartKeys.iterator(); Iterator<FieldSchema> newPartKeysIter = newPartKeys.iterator(); FieldSchema oldFs; FieldSchema newFs; while (oldPartKeysIter.hasNext()) { oldFs = oldPartKeysIter.next(); newFs = newPartKeysIter.next(); // Alter table can change the type of partition key now. // So check the column name only. if (!oldFs.getName().equals(newFs.getName())) { return false; } } return true; }
/** * Generate normalized name for a given target column. */ private String getTargetFieldName(int fieldIndex, String destTableName, List<String> colNames, List<FieldSchema> fieldSchemas) { String fieldName = fieldSchemas.get(fieldIndex).getName(); String[] parts = fieldName.split("\\."); if (destTableName != null) { String colName = parts[parts.length - 1]; if (colNames != null && !colNames.contains(colName)) { colName = colNames.get(fieldIndex); } return destTableName + "." + colName; } if (parts.length == 2 && parts[0].startsWith("_u")) { return parts[1]; } return fieldName; }
private List<Integer> getBucketColIDs(List<String> bucketCols, List<FieldSchema> cols) { ArrayList<Integer> result = new ArrayList<Integer>(bucketCols.size()); HashSet<String> bucketSet = new HashSet<String>(bucketCols); for (int i = 0; i < cols.size(); i++) { if( bucketSet.contains(cols.get(i).getName()) ) { result.add(i); } } return result; }
/** * Given a list of partition columns and a partial mapping from * some partition columns to values the function returns the values * for the column. * @param partCols the list of table partition columns * @param partSpec the partial mapping from partition column to values * @return list of values of for given partition columns, any missing * values in partSpec is replaced by an empty string */ public static List<String> getPvals(List<FieldSchema> partCols, Map<String, String> partSpec) { List<String> pvals = new ArrayList<>(partCols.size()); for (FieldSchema field : partCols) { String val = StringUtils.defaultString(partSpec.get(field.getName())); pvals.add(val); } return pvals; } public static String makePartNameMatcher(Table table, List<String> partVals, String defaultStr) throws MetaException {
private List<Integer> getBucketPositions(List<String> tabBucketCols, List<FieldSchema> tabCols) { List<Integer> posns = new ArrayList<Integer>(); for (String bucketCol : tabBucketCols) { int pos = 0; for (FieldSchema tabCol : tabCols) { if (bucketCol.equals(tabCol.getName())) { posns.add(pos); break; } pos++; } } return posns; }
public static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel, Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema) throws CalciteSemanticException { RelNode parentOforiginalProjRel = topSelparentPair.getKey(); HiveProject originalProjRel = (HiveProject) topSelparentPair.getValue(); // Assumption: top portion of tree could only be // (limit)?(OB)?(Project).... List<RexNode> rootChildExps = originalProjRel.getChildExps(); if (resultSchema.size() != rootChildExps.size()) { // Safeguard against potential issues in CBO RowResolver construction. Disable CBO for now. LOG.error(PlanModifierUtil.generateInvalidSchemaMessage(originalProjRel, resultSchema, 0)); throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema"); } List<String> newSelAliases = new ArrayList<String>(); String colAlias; for (int i = 0; i < rootChildExps.size(); i++) { colAlias = resultSchema.get(i).getName(); colAlias = getNewColAlias(newSelAliases, colAlias); newSelAliases.add(colAlias); } HiveProject replacementProjectRel = HiveProject.create(originalProjRel.getInput(), originalProjRel.getChildExps(), newSelAliases); if (rootRel == originalProjRel) { return replacementProjectRel; } else { parentOforiginalProjRel.replaceInput(0, replacementProjectRel); return rootRel; } }
protected List<Integer> getBucketColIDs(List<String> bucketCols, List<FieldSchema> cols) { ArrayList<Integer> result = new ArrayList<>(bucketCols.size()); HashSet<String> bucketSet = new HashSet<>(bucketCols); for (int i = 0; i < cols.size(); i++) { if (bucketSet.contains(cols.get(i).getName())) { result.add(i); } } return result; }
public static String validateSkewedColNamesSubsetCol(List<String> skewedColNames, List<FieldSchema> cols) { if (CollectionUtils.isEmpty(skewedColNames)) { return null; } List<String> colNames = new ArrayList<>(cols.size()); for (FieldSchema fieldSchema : cols) { colNames.add(fieldSchema.getName()); } // make a copy List<String> copySkewedColNames = new ArrayList<>(skewedColNames); // remove valid columns copySkewedColNames.removeAll(colNames); if (copySkewedColNames.isEmpty()) { return null; } return copySkewedColNames.toString(); }