private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs) throws IOException { FileStatus status = getReferencedFileStatus(fs); if (this.reference != null) { return computeRefFileHDFSBlockDistribution(fs, reference, status); } else { return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); } }
/** * helper function to compute HDFS blocks distribution of a given reference * file.For reference file, we don't compute the exact value. We use some * estimate instead given it might be good enough. we assume bottom part * takes the first half of reference file, top part takes the second half * of the reference file. This is just estimate, given * midkey ofregion != midkey of HFile, also the number and size of keys vary. * If this estimate isn't good enough, we can improve it later. * @param fs The FileSystem * @param reference The reference * @param status The reference FileStatus * @return HDFS blocks distribution */ private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution( final FileSystem fs, final Reference reference, final FileStatus status) throws IOException { if (status == null) { return null; } long start = 0; long length = 0; if (Reference.isTopFileRegion(reference.getFileRegion())) { start = status.getLen()/2; length = status.getLen() - status.getLen()/2; } else { start = 0; length = status.getLen()/2; } return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length); }
/** * return the top hosts of the store files, used by the Split */ private static String[] getStoreDirHosts(final FileSystem fs, final Path path) throws IOException { FileStatus[] files = FSUtils.listStatus(fs, path); if (files == null) { return new String[] {}; } HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); for (FileStatus hfileStatus: files) { HDFSBlocksDistribution storeFileBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } List<String> hosts = hdfsBlocksDistribution.getTopHosts(); return hosts.toArray(new String[hosts.size()]); }
FileStatus status = fs.getFileStatus(testFile); HDFSBlocksDistribution blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); long uniqueBlocksTotalWeight = blocksDistribution.getUniqueBlocksTotalWeight(); FileStatus status = fs.getFileStatus(testFile); HDFSBlocksDistribution blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); uniqueBlocksTotalWeight = blocksDistribution.getUniqueBlocksTotalWeight(); do { FileStatus status = fs.getFileStatus(testFile); blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs) throws IOException { FileStatus status = getReferencedFileStatus(fs); if (this.reference != null) { return computeRefFileHDFSBlockDistribution(fs, reference, status); } else { return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); } }
/** * helper function to compute HDFS blocks distribution of a given reference * file.For reference file, we don't compute the exact value. We use some * estimate instead given it might be good enough. we assume bottom part * takes the first half of reference file, top part takes the second half * of the reference file. This is just estimate, given * midkey ofregion != midkey of HFile, also the number and size of keys vary. * If this estimate isn't good enough, we can improve it later. * @param fs The FileSystem * @param reference The reference * @param status The reference FileStatus * @return HDFS blocks distribution */ static private HDFSBlocksDistribution computeRefFileHDFSBlockDistribution( FileSystem fs, Reference reference, FileStatus status) throws IOException { if (status == null) { return null; } long start = 0; long length = 0; if (Reference.isTopFileRegion(reference.getFileRegion())) { start = status.getLen()/2; length = status.getLen() - status.getLen()/2; } else { start = 0; length = status.getLen()/2; } return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length); }
/** * helper function to compute HDFS blocks distribution of a given reference * file.For reference file, we don't compute the exact value. We use some * estimate instead given it might be good enough. we assume bottom part * takes the first half of reference file, top part takes the second half * of the reference file. This is just estimate, given * midkey ofregion != midkey of HFile, also the number and size of keys vary. * If this estimate isn't good enough, we can improve it later. * @param fs The FileSystem * @param reference The reference * @param status The reference FileStatus * @return HDFS blocks distribution */ private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution( final FileSystem fs, final Reference reference, final FileStatus status) throws IOException { if (status == null) { return null; } long start = 0; long length = 0; if (Reference.isTopFileRegion(reference.getFileRegion())) { start = status.getLen()/2; length = status.getLen() - status.getLen()/2; } else { start = 0; length = status.getLen()/2; } return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length); }
/** * return the top hosts of the store files, used by the Split */ private static String[] getStoreDirHosts(final FileSystem fs, final Path path) throws IOException { FileStatus[] files = FSUtils.listStatus(fs, path); if (files == null) { return new String[] {}; } HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); for (FileStatus hfileStatus: files) { HDFSBlocksDistribution storeFileBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } List<String> hosts = hdfsBlocksDistribution.getTopHosts(); return hosts.toArray(new String[hosts.size()]); }
/** * return the top hosts of the store files, used by the Split */ private static String[] getStoreDirHosts(final FileSystem fs, final Path path) throws IOException { FileStatus[] files = FSUtils.listStatus(fs, path); if (files == null) { return new String[] {}; } HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); for (FileStatus hfileStatus: files) { HDFSBlocksDistribution storeFileBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } List<String> hosts = hdfsBlocksDistribution.getTopHosts(); return hosts.toArray(new String[hosts.size()]); }
/** * return the top hosts of the store files, used by the Split */ private static String[] getStoreDirHosts(final FileSystem fs, final Path path) throws IOException { FileStatus[] files = FSUtils.listStatus(fs, path, null); if (files == null) { return new String[] {}; } HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); for (FileStatus hfileStatus: files) { HDFSBlocksDistribution storeFileBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } List<String> hosts = hdfsBlocksDistribution.getTopHosts(); return hosts.toArray(new String[hosts.size()]); }
/** * return the top hosts of the store files, used by the Split */ private static String[] getStoreDirHosts(final FileSystem fs, final Path path) throws IOException { FileStatus[] files = FSUtils.listStatus(fs, path); if (files == null) { return new String[] {}; } HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); for (FileStatus hfileStatus: files) { HDFSBlocksDistribution storeFileBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } List<String> hosts = hdfsBlocksDistribution.getTopHosts(); return hosts.toArray(new String[hosts.size()]); }
/** * compute HDFS block distribution, for reference file, it is an estimate */ private void computeHDFSBlockDistribution() throws IOException { if (isReference()) { FileStatus status; if (this.link != null) { status = this.link.getFileStatus(fs); } else { status = fs.getFileStatus(this.referencePath); } this.hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution( this.fs, this.reference, status); } else { FileStatus status; if (isLink()) { status = link.getFileStatus(fs); } else { status = this.fs.getFileStatus(path); } long length = status.getLen(); this.hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution( this.fs, status, 0, length); } }
FSUtils.computeHDFSBlocksDistribution(fs, hfileStatus, 0, hfileStatus.getLen()); hdfsBlocksDistribution.add(storeFileBlocksDistribution);
FileStatus status = fs.getFileStatus(testFile); HDFSBlocksDistribution blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); long uniqueBlocksTotalWeight = blocksDistribution.getUniqueBlocksTotalWeight(); FileStatus status = fs.getFileStatus(testFile); HDFSBlocksDistribution blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen()); uniqueBlocksTotalWeight = blocksDistribution.getUniqueBlocksTotalWeight(); do { FileStatus status = fs.getFileStatus(testFile); blocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());