/** * Match column in skewed column list and record position. * The position will be used in {@link FileSinkOperator} generateListBucketingDirName(). * Note that skewed column name matches skewed value in order. * * @param rowSch */ public void processRowSkewedIndex(RowSchema rowSch) { if ((this.skewedColNames != null) && (this.skewedColNames.size() > 0) && (rowSch != null) && (rowSch.getSignature() != null) && (rowSch.getSignature().size() > 0)) { List<ColumnInfo> cols = rowSch.getSignature(); int hitNo = 0; for (int i = 0; i < cols.size(); i++) { int index = this.skewedColNames.indexOf(cols.get(i).getInternalName()); if (index > -1) { hitNo++; SkewedColumnPositionPair pair = new SkewedColumnPositionPair(i, index); rowSkewedIndex.add(pair); } } assert (hitNo == this.skewedColNames.size()) : "RowSchema doesn't have all skewed columns." + "Skewed column: " + this.skewedColNames.toString() + ". Rowschema has columns: " + cols; } }
skewedValsCandidate.add(posPair.getSkewColPosition(), standObjs.get(posPair.getTblColPosition()).toString());
skewedValsCandidate.add(posPair.getSkewColPosition(), standObjs.get(posPair.getTblColPosition()).toString());
/** * Match column in skewed column list and record position. * The position will be used in {@link FileSinkOperator} generateListBucketingDirName(). * Note that skewed column name matches skewed value in order. * * @param rowSch */ public void processRowSkewedIndex(RowSchema rowSch) { if ((this.skewedColNames != null) && (this.skewedColNames.size() > 0) && (rowSch != null) && (rowSch.getSignature() != null) && (rowSch.getSignature().size() > 0)) { List<ColumnInfo> cols = rowSch.getSignature(); int hitNo = 0; for (int i = 0; i < cols.size(); i++) { int index = this.skewedColNames.indexOf(cols.get(i).getInternalName()); if (index > -1) { hitNo++; SkewedColumnPositionPair pair = new SkewedColumnPositionPair(i, index); rowSkewedIndex.add(pair); } } assert (hitNo == this.skewedColNames.size()) : "RowSchema doesn't have all skewed columns." + "Skewed column: " + this.skewedColNames.toString() + ". Rowschema has columns: " + cols; } }
skewedValsCandidate.add(posPair.getSkewColPosition(), standObjs.get(posPair.getTblColPosition()).toString());
/** * Match column in skewed column list and record position. * The position will be used in {@link FileSinkOperator} generateListBucketingDirName(). * Note that skewed column name matches skewed value in order. * * @param rowSch */ public void processRowSkewedIndex(RowSchema rowSch) { if ((this.skewedColNames != null) && (this.skewedColNames.size() > 0) && (rowSch != null) && (rowSch.getSignature() != null) && (rowSch.getSignature().size() > 0)) { List<ColumnInfo> cols = rowSch.getSignature(); int hitNo = 0; for (int i = 0; i < cols.size(); i++) { int index = this.skewedColNames.indexOf(cols.get(i).getInternalName()); if (index > -1) { hitNo++; SkewedColumnPositionPair pair = new SkewedColumnPositionPair(i, index); rowSkewedIndex.add(pair); } } assert (hitNo == this.skewedColNames.size()) : "RowSchema doesn't have all skewed columns." + "Skewed column: " + this.skewedColNames.toString() + ". Rowschema has columns: " + cols; } }