/** * get all paths for this partition in a sorted manner */ @SuppressWarnings("nls") public FileStatus[] getSortedPaths() { try { // Previously, this got the filesystem of the Table, which could be // different from the filesystem of the partition. FileSystem fs = getDataLocation().getFileSystem(SessionState.getSessionConf()); String pathPattern = getDataLocation().toString(); if (getBucketCount() > 0) { pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); } if (srcs.length == 0) { return null; } return srcs; } catch (Exception e) { throw new RuntimeException("Cannot get path ", e); } }
/** * get all paths for this partition in a sorted manner */ @SuppressWarnings("nls") public FileStatus[] getSortedPaths() { try { // Previously, this got the filesystem of the Table, which could be // different from the filesystem of the partition. FileSystem fs = getDataLocation().getFileSystem(SessionState.getSessionConf()); String pathPattern = getDataLocation().toString(); if (getBucketCount() > 0) { pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); } if (srcs.length == 0) { return null; } return srcs; } catch (Exception e) { throw new RuntimeException("Cannot get path ", e); } }
getBucketFilePathsOfPartition(p.getDataLocation(), pGraphContext); int bucketCount = p.getBucketCount(); p.getBucketCount() + ", whereas the number of files is " + fileNames.size(); throw new SemanticException( ErrorMsg.BUCKETED_TABLE_METADATA_INCORRECT.getMsg(msg));
getBucketFilePathsOfPartition(p.getDataLocation(), pGraphContext); int bucketCount = p.getBucketCount(); p.getBucketCount() + ", whereas the number of files is " + fileNames.size(); throw new SemanticException( ErrorMsg.BUCKETED_TABLE_METADATA_INCORRECT.getMsg(msg));
private boolean checkPartition(Partition partition, List<Integer> bucketPositionsDest, List<Integer> sortPositionsDest, List<Integer> sortOrderDest, int numBucketsDest) { // The bucketing and sorting positions should exactly match int numBuckets = partition.getBucketCount(); if (numBucketsDest != numBuckets) { return false; } List<Integer> partnBucketPositions = getBucketPositions(partition.getBucketCols(), partition.getTable().getCols()); List<Integer> sortPositions = getSortPositions(partition.getSortCols(), partition.getTable().getCols()); List<Integer> sortOrder = getSortOrder(partition.getSortCols(), partition.getTable().getCols()); return bucketPositionsDest.equals(partnBucketPositions) && sortPositionsDest.equals(sortPositions) && sortOrderDest.equals(sortOrder); }
int num = sampleDescr.getNumerator(); int den = sampleDescr.getDenominator(); int bucketCount = part.getBucketCount(); String fullScanMsg = "";
int num = sampleDescr.getNumerator(); int den = sampleDescr.getDenominator(); int bucketCount = part.getBucketCount(); String fullScanMsg = "";
return getPath(); } else { int bcount = getBucketCount(); if (bcount == 0) { return getPath();
private boolean checkPartition(Partition partition, List<Integer> bucketPositionsDest, List<Integer> sortPositionsDest, List<Integer> sortOrderDest, int numBucketsDest) { // The bucketing and sorting positions should exactly match int numBuckets = partition.getBucketCount(); if (numBucketsDest != numBuckets) { return false; } List<Integer> partnBucketPositions = getBucketPositions(partition.getBucketCols(), partition.getTable().getCols()); List<Integer> sortPositions = getSortPositions(partition.getSortCols(), partition.getTable().getCols()); List<Integer> sortOrder = getSortOrder(partition.getSortCols(), partition.getTable().getCols()); return bucketPositionsDest.equals(partnBucketPositions) && sortPositionsDest.equals(sortPositions) && sortOrderDest.equals(sortOrder); }
return getPath(); } else { int bcount = getBucketCount(); if (bcount == 0) { return getPath();
if (prunedPartList != null) { for (Partition p : prunedPartList.getPartitions()) { if (numBuckets != p.getBucketCount()) {
ctxt.setPartitions(prunedPartList); for (Partition p : prunedPartList.getPartitions()) { if (nbuckets != p.getBucketCount()) {
} else if (alterTbl.getOp() == AlterTableTypes.ALTERBUCKETNUM) { if (part != null) { if (part.getBucketCount() == alterTbl.getNumberBuckets()) { return 0;
} else if (alterTbl.getOp() == AlterTableTypes.ALTERBUCKETNUM) { if (part != null) { if (part.getBucketCount() == alterTbl.getNumberBuckets()) { return null;
/** * mapping from bucket number to bucket path */ // TODO: add test case and clean it up @SuppressWarnings("nls") public Path getBucketPath(int bucketNum) { try { // Previously, this got the filesystem of the Table, which could be // different from the filesystem of the partition. FileSystem fs = FileSystem.get(getPartitionPath().toUri(), Hive.get() .getConf()); String pathPattern = getPartitionPath().toString(); if (getBucketCount() > 0) { pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern)); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); } if (srcs.length == 0) { return null; } return srcs[bucketNum].getPath(); } catch (Exception e) { throw new RuntimeException("Cannot get bucket path for bucket " + bucketNum, e); } }
/** * get all paths for this partition in a sorted manner */ @SuppressWarnings("nls") public FileStatus[] getSortedPaths() { try { // Previously, this got the filesystem of the Table, which could be // different from the filesystem of the partition. FileSystem fs = getDataLocation().getFileSystem(SessionState.getSessionConf()); String pathPattern = getDataLocation().toString(); if (getBucketCount() > 0) { pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); } if (srcs.length == 0) { return null; } return srcs; } catch (Exception e) { throw new RuntimeException("Cannot get path ", e); } }
private void assertSamePartitions(List<Partition> parts1, List<Partition> parts2) { assertEquals(parts1.size(), parts2.size()); for (int i = 0; i < parts1.size(); i++) { assertEquals(parts1.get(i).toString(), parts2.get(i).toString(), "toString differs at element " + i); assertEquals(parts1.get(i).getPath(), parts2.get(i).getPath(), "path differs at element " + i); assertEquals(parts1.get(i).getLocation(), parts2.get(i).getLocation(), "location differs at element " + i); assertEquals(parts1.get(i).getBucketCount(), parts2.get(i).getBucketCount(), "bucket count differs at element " + i); assertEquals(parts1.get(i).getCompleteName(), parts2.get(i).getCompleteName(), "complete name differs at element " + i); } }
public boolean checkBucketedTable(Table tbl, ParseContext pGraphContext, PrunedPartitionList prunedParts) throws SemanticException { if (tbl.isPartitioned()) { List<Partition> partitions = prunedParts.getNotDeniedPartns(); // construct a mapping of (Partition->bucket file names) and (Partition -> bucket number) if (!partitions.isEmpty()) { for (Partition p : partitions) { List<String> fileNames = AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(), pGraphContext); // The number of files for the table should be same as number of // buckets. int bucketCount = p.getBucketCount(); if (fileNames.size() != 0 && fileNames.size() != bucketCount) { return false; } } } } else { List<String> fileNames = AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(), pGraphContext); Integer num = new Integer(tbl.getNumBuckets()); // The number of files for the table should be same as number of buckets. if (fileNames.size() != 0 && fileNames.size() != num) { return false; } } return true; }
private boolean checkPartition(Partition partition, List<Integer> bucketPositionsDest, List<Integer> sortPositionsDest, List<Integer> sortOrderDest, int numBucketsDest) { // The bucketing and sorting positions should exactly match int numBuckets = partition.getBucketCount(); if (numBucketsDest != numBuckets) { return false; } List<Integer> partnBucketPositions = getBucketPositions(partition.getBucketCols(), partition.getTable().getCols()); ObjectPair<List<Integer>, List<Integer>> partnSortPositionsOrder = getSortPositionsOrder(partition.getSortCols(), partition.getTable().getCols()); return bucketPositionsDest.equals(partnBucketPositions) && sortPositionsDest.equals(partnSortPositionsOrder.getFirst()) && sortOrderDest.equals(partnSortPositionsOrder.getSecond()); }
return getPath(); } else { int bcount = getBucketCount(); if (bcount == 0) { return getPath();