/** * Initialize list bucketing information */ private void lbSetup() { this.isSkewedStoredAsSubDirectories = ((lbCtx == null) ? false : lbCtx.isSkewedStoredAsDir()); }
/** * Initialize list bucketing information */ private void lbSetup() { this.isSkewedStoredAsSubDirectories = ((lbCtx == null) ? false : lbCtx.isSkewedStoredAsDir()); }
/** * Calculate list bucketing level. * * 0: not list bucketing * int: no. of skewed columns * * @param ctx * @return */ public int calculateListBucketingLevel() { int lbLevel = isSkewedStoredAsDir() ? this.getSkewedColNames().size() : 0; return lbLevel; }
/** * Calculate list bucketing level. * * 0: not list bucketing * int: no. of skewed columns * * @param ctx * @return */ public int calculateListBucketingLevel() { int lbLevel = isSkewedStoredAsDir() ? this.getSkewedColNames().size() : 0; return lbLevel; }
/** * check if it is skewed table and stored as dirs. * * @param fsInputDesc * @return */ public static boolean isSkewedStoredAsDirs(FileSinkDesc fsInputDesc) { return (fsInputDesc.getLbCtx() == null) ? false : fsInputDesc.getLbCtx() .isSkewedStoredAsDir(); }
private boolean isSkewedStoredAsDirs(LoadTableDesc tbd) { return (tbd.getLbCtx() == null) ? false : tbd.getLbCtx() .isSkewedStoredAsDir(); }
/** * Calculate skewed value subdirectory directory which is used in * FileSinkOperator.java createKeyForStatsPublisher() * For example, create table test skewed by (key, value) on (('484','val_484') * stored as DIRECTORIES; * after the method, skewedValuesDirNames will contain 2 elements: * key=484/value=val_484 * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME */ public void calculateSkewedValueSubDirList() { if (isSkewedStoredAsDir()) { for (List<String> value : this.skewedColValues) { skewedValuesDirNames.add(FileUtils.makeListBucketingDirName(this.skewedColNames, value)); } // creat default dir skewedValuesDirNames.add(FileUtils.makeDefaultListBucketingDirName( this.skewedColNames, ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)); } }
/** * check if it is skewed table and stored as dirs. * * @param fsInputDesc * @return */ public static boolean isSkewedStoredAsDirs(FileSinkDesc fsInputDesc) { return (fsInputDesc.getLbCtx() == null) ? false : fsInputDesc.getLbCtx() .isSkewedStoredAsDir(); }
private boolean isSkewedStoredAsDirs(LoadTableDesc tbd) { return (tbd.getLbCtx() == null) ? false : tbd.getLbCtx() .isSkewedStoredAsDir(); }
/** * Calculate skewed value subdirectory directory which is used in * FileSinkOperator.java createKeyForStatsPublisher() * For example, create table test skewed by (key, value) on (('484','val_484') * stored as DIRECTORIES; * after the method, skewedValuesDirNames will contain 2 elements: * key=484/value=val_484 * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME */ public void calculateSkewedValueSubDirList() { if (isSkewedStoredAsDir()) { for (List<String> value : this.skewedColValues) { skewedValuesDirNames.add(FileUtils.makeListBucketingDirName(this.skewedColNames, value)); } // creat default dir skewedValuesDirNames.add(FileUtils.makeDefaultListBucketingDirName( this.skewedColNames, ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)); } }
isListBucketingAlterTableConcatenate = ((listBucketingCtx == null) ? false : listBucketingCtx .isSkewedStoredAsDir()); LOG.info("isListBucketingAlterTableConcatenate : " + isListBucketingAlterTableConcatenate);
isListBucketingAlterTableConcatenate = ((listBucketingCtx == null) ? false : listBucketingCtx .isSkewedStoredAsDir()); LOG.info("isListBucketingAlterTableConcatenate : " + isListBucketingAlterTableConcatenate);
DriverContext driverContext) throws HiveException { ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx(); boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
DriverContext driverContext) throws HiveException { ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx(); boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
/** * Initialize list bucketing information */ private void lbSetup() { this.isSkewedStoredAsSubDirectories = ((lbCtx == null) ? false : lbCtx.isSkewedStoredAsDir()); }
/** * Calculate list bucketing level. * * 0: not list bucketing * int: no. of skewed columns * * @param ctx * @return */ public int calculateListBucketingLevel() { int lbLevel = isSkewedStoredAsDir() ? this.getSkewedColNames().size() : 0; return lbLevel; }
/** * check if it is skewed table and stored as dirs. * * @param fsInputDesc * @return */ public static boolean isSkewedStoredAsDirs(FileSinkDesc fsInputDesc) { return (fsInputDesc.getLbCtx() == null) ? false : fsInputDesc.getLbCtx() .isSkewedStoredAsDir(); }
private boolean isSkewedStoredAsDirs(LoadTableDesc tbd) { return (tbd.getLbCtx() == null) ? false : tbd.getLbCtx() .isSkewedStoredAsDir(); }
/** * Calculate skewed value subdirectory directory which is used in * FileSinkOperator.java createKeyForStatsPublisher() * For example, create table test skewed by (key, value) on (('484','val_484') * stored as DIRECTORIES; * after the method, skewedValuesDirNames will contain 2 elements: * key=484/value=val_484 * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME */ public void calculateSkewedValueSubDirList() { if (isSkewedStoredAsDir()) { for (List<String> value : this.skewedColValues) { skewedValuesDirNames.add(FileUtils.makeListBucketingDirName(this.skewedColNames, value)); } // creat default dir skewedValuesDirNames.add(FileUtils.makeDefaultListBucketingDirName( this.skewedColNames, ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)); } }
isListBucketingAlterTableConcatenate = ((listBucketingCtx == null) ? false : listBucketingCtx .isSkewedStoredAsDir()); LOG.info("isListBucketingAlterTableConcatenate : " + isListBucketingAlterTableConcatenate);