private static long getDataSize(HiveConf conf, Table table) { long ds = getRawDataSize(table); if (ds <= 0) { ds = getTotalSize(table); // if data size is still 0 then get file size if (ds <= 0) { ds = getFileSizeForTable(conf, table); } float deserFactor = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_DESERIALIZATION_FACTOR); ds = (long) (ds * deserFactor); } return ds; }
/** * Returns the size of the table in bytes, does not take into consideration filter/partition * details passed, if any. */ @Override public long getEstimatedSizeBytes(PipelineOptions pipelineOptions) throws Exception { Configuration conf = new Configuration(); for (Entry<String, String> entry : spec.getConfigProperties().entrySet()) { conf.set(entry.getKey(), entry.getValue()); } IMetaStoreClient client = null; try { HiveConf hiveConf = HCatUtil.getHiveConf(conf); client = HCatUtil.getHiveMetastoreClient(hiveConf); Table table = HCatUtil.getTable(client, spec.getDatabase(), spec.getTable()); return StatsUtils.getFileSizeForTable(hiveConf, table); } finally { // IMetaStoreClient is not AutoCloseable, closing it manually if (client != null) { client.close(); } } }
ds = getFileSizeForTable(conf, table);