protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames, List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps, boolean isStoredAsSubDirectories) { ListBucketingCtx lbCtx = new ListBucketingCtx(); lbCtx.setSkewedColNames(skewedColNames); lbCtx.setSkewedColValues(skewedValues); lbCtx.setLbLocationMap(skewedColValueLocationMaps); lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories); lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY); lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME); return lbCtx; }
List<String> skewedCols = lbCtx.getSkewedColNames(); List<List<String>> allSkewedVals = lbCtx.getSkewedColValues(); List<String> skewedValsCandidate = null; Map<List<String>, String> locationMap = lbCtx.getLbLocationMap(); for (SkewedColumnPositionPair posPair : lbCtx.getRowSkewedIndex()) { skewedValsCandidate.add(posPair.getSkewColPosition(), standObjs.get(posPair.getTblColPosition()).toString()); lbCtx.getDefaultDirName()); List<String> defaultKey = Lists.newArrayList(lbCtx.getDefaultKey()); if (!locationMap.containsKey(defaultKey)) { locationMap.put(defaultKey, lbDirName);
for (String dir : lbCtx.getSkewedValuesDirNames()) { int index = fspKey.indexOf(dir); if (index >= 0) {
/** * check if list bucketing is enabled. * * @param ctx * @return */ public boolean isSkewedStoredAsDir() { return (this.getSkewedColNames() != null) && (this.getSkewedColNames().size() > 0) && (this.getSkewedColValues() != null) && (this.getSkewedColValues().size() > 0) && (this.isStoredAsSubDirectories()); }
/** * Calculate list bucketing level. * * 0: not list bucketing * int: no. of skewed columns * * @param ctx * @return */ public int calculateListBucketingLevel() { int lbLevel = isSkewedStoredAsDir() ? this.getSkewedColNames().size() : 0; return lbLevel; }
List<String> skewedCols = lbCtx.getSkewedColNames(); List<List<String>> allSkewedVals = lbCtx.getSkewedColValues(); Map<List<String>, String> locationMap = lbCtx.getLbLocationMap(); for (SkewedColumnPositionPair posPair : lbCtx.getRowSkewedIndex()) { skewedValsCandidate.add(posPair.getSkewColPosition(), standObjs.get(posPair.getTblColPosition()).toString());
DriverContext driverContext) throws HiveException { ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx(); boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
lbCtx.processRowSkewedIndex(fsRS); lbCtx.calculateSkewedValueSubDirList();
if (lbCtx != null && !lbCtx.getSkewedColNames().isEmpty() && !lbCtx.getSkewedColValues().isEmpty()) { LOG.debug("Bailing out of sort dynamic partition optimization as list bucketing is enabled"); return null;
int lbLevel = (ctx.getLbCtx() == null) ? 0 : ctx.getLbCtx().calculateListBucketingLevel();
private String createDefaultLbDir(List<String> skewedCols, Map<List<String>, String> locationMap) { String lbDirName; lbDirName = FileUtils.makeDefaultListBucketingDirName(skewedCols, lbCtx.getDefaultDirName()); List<String> defaultKey = Lists.newArrayList(lbCtx.getDefaultKey()); if (!locationMap.containsKey(defaultKey)) { locationMap.put(defaultKey, lbDirName); } return lbDirName; }
/** * check if list bucketing is enabled. * * @param ctx * @return */ public boolean isSkewedStoredAsDir() { return (this.getSkewedColNames() != null) && (this.getSkewedColNames().size() > 0) && (this.getSkewedColValues() != null) && (this.getSkewedColValues().size() > 0) && (this.isStoredAsSubDirectories()); }
/** * Calculate list bucketing level. * * 0: not list bucketing * int: no. of skewed columns * * @param ctx * @return */ public int calculateListBucketingLevel() { int lbLevel = isSkewedStoredAsDir() ? this.getSkewedColNames().size() : 0; return lbLevel; }
DriverContext driverContext) throws HiveException { ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx(); boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
lbCtx.processRowSkewedIndex(fsRS); lbCtx.calculateSkewedValueSubDirList();
if (lbCtx != null && !lbCtx.getSkewedColNames().isEmpty() && !lbCtx.getSkewedColValues().isEmpty()) { LOG.debug("Bailing out of sort dynamic partition optimization as list bucketing is enabled"); return null;
int lbLevel = (ctx.getLbCtx() == null) ? 0 : ctx.getLbCtx().calculateListBucketingLevel();
/** * Construct list bucketing context. * * @param skewedColNames * @param skewedValues * @param skewedColValueLocationMaps * @param isStoredAsSubDirectories * @return */ protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames, List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps, boolean isStoredAsSubDirectories, HiveConf conf) { ListBucketingCtx lbCtx = new ListBucketingCtx(); lbCtx.setSkewedColNames(skewedColNames); lbCtx.setSkewedColValues(skewedValues); lbCtx.setLbLocationMap(skewedColValueLocationMaps); lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories); lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY); lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME); return lbCtx; }
List<String> skewedCols = lbCtx.getSkewedColNames(); List<List<String>> allSkewedVals = lbCtx.getSkewedColValues(); List<String> skewedValsCandidate = null; Map<List<String>, String> locationMap = lbCtx.getLbLocationMap(); for (SkewedColumnPositionPair posPair : lbCtx.getRowSkewedIndex()) { skewedValsCandidate.add(posPair.getSkewColPosition(), standObjs.get(posPair.getTblColPosition()).toString()); lbCtx.getDefaultDirName()); List<String> defaultKey = Arrays.asList(lbCtx.getDefaultKey()); if (!locationMap.containsKey(defaultKey)) { locationMap.put(defaultKey, lbDirName);
/** * check if list bucketing is enabled. * * @param ctx * @return */ public boolean isSkewedStoredAsDir() { return (this.getSkewedColNames() != null) && (this.getSkewedColNames().size() > 0) && (this.getSkewedColValues() != null) && (this.getSkewedColValues().size() > 0) && (this.isStoredAsSubDirectories()); }