private static Map<String, Object> toJsonMap( final BlockLocation blockLocation) throws IOException { if (blockLocation == null) { return null; } final Map<String, Object> m = new HashMap<>(); m.put("length", blockLocation.getLength()); m.put("offset", blockLocation.getOffset()); m.put("corrupt", blockLocation.isCorrupt()); m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes())); m.put("cachedHosts", blockLocation.getCachedHosts()); m.put("hosts", blockLocation.getHosts()); m.put("names", blockLocation.getNames()); m.put("topologyPaths", blockLocation.getTopologyPaths()); return m; }
public static Locality locality(FileStatus fstat, BlockLocation[] blocks) throws IOException { Path p = fstat.getPath(); Integer numBlocks = blocks.length; Locality l = new Locality(p, numBlocks); for (int i = 0; i < blocks.length; i++) { final BlockLocation block = blocks[i]; final String[] hosts = block.getHosts(); final String[] cached = block.getCachedHosts(); for (String host : hosts) { List<Integer> blockList = l.hostBlocks.get(host); if (blockList == null) { blockList = new ArrayList<Integer>(); l.hostBlocks.put(host, blockList); } blockList.add(i); l.hosts.add(host); } for (String host : cached) { List<Integer> bl = l.cachedBlocks.get(host); if (bl == null) { bl = new ArrayList<Integer>(); l.cachedBlocks.put(host, bl); } bl.add(i); l.hosts.add(host); } } return l; }
System.out.println(" Length: " + blockLocation.getLength()); String[] cachedHosts = blockLocation.getCachedHosts(); if (cachedHosts.length == 0) { System.out.println(" No cached hosts");
long offset = length - bytesRemaining; int blkIndex = getBlockIndex(blkLocations, offset); splits.add(makeSplit(path, offset, splitSize, splitIndex++, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); bytesRemaining -= splitSize; long offset = length - bytesRemaining; int blkIndex = getBlockIndex(blkLocations, offset); splits.add(makeSplit(path, offset, bytesRemaining, splitIndex++, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); splits.add(makeSplit(path, skipLength, length - skipLength, splitIndex++, blkLocations[0].getHosts(), blkLocations[0].getCachedHosts()));
splits.add(makeSplit(path, length-bytesRemaining, splitSize, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); bytesRemaining -= splitSize; splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); blkLocations[0].getCachedHosts()));
splits.add(makeSplit(path, length-bytesRemaining, splitSize, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); bytesRemaining -= splitSize; splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); blkLocations[0].getCachedHosts()));
splits.add(makeSplit(path, length-bytesRemaining, splitSize, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); bytesRemaining -= splitSize; splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); blkLocations[0].getCachedHosts()));
long offset = length - bytesRemaining; int blkIndex = getBlockIndex(blkLocations, offset); splits.add(makeSplit(path, offset, splitSize, splitIndex++, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); bytesRemaining -= splitSize; long offset = length - bytesRemaining; int blkIndex = getBlockIndex(blkLocations, offset); splits.add(makeSplit(path, offset, bytesRemaining, splitIndex++, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); splits.add(makeSplit(path, skipLength, length - skipLength, splitIndex++, blkLocations[0].getHosts(), blkLocations[0].getCachedHosts()));
splits.add(makeSplit(path, length-bytesRemaining, splitSize, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); bytesRemaining -= splitSize; splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts())); blkLocations[0].getCachedHosts()));
private static void checkNumCachedReplicas(final DistributedFileSystem dfs, final List<Path> paths, final int expectedBlocks, final int expectedReplicas) throws Exception { int numCachedBlocks = 0; int numCachedReplicas = 0; for (Path p: paths) { final FileStatus f = dfs.getFileStatus(p); final long len = f.getLen(); final long blockSize = f.getBlockSize(); // round it up to full blocks final long numBlocks = (len + blockSize - 1) / blockSize; BlockLocation[] locs = dfs.getFileBlockLocations(p, 0, len); assertEquals("Unexpected number of block locations for path " + p, numBlocks, locs.length); for (BlockLocation l: locs) { if (l.getCachedHosts().length > 0) { numCachedBlocks++; } numCachedReplicas += l.getCachedHosts().length; } } LOG.info("Found " + numCachedBlocks + " of " + expectedBlocks + " blocks"); LOG.info("Found " + numCachedReplicas + " of " + expectedReplicas + " replicas"); assertEquals("Unexpected number of cached blocks", expectedBlocks, numCachedBlocks); assertEquals("Unexpected number of cached replicas", expectedReplicas, numCachedReplicas); }
makeSplit(streamFile.getSegmentNo(), path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1)); bytesRemaining -= splitSize; splits.add(makeSplit(streamFile.getSegmentNo(), path, length - bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1));
blkLocations[startIndex].getCachedHosts() };
blkLocations[startIndex].getCachedHosts() };
blkLocations[startIndex].getCachedHosts() };
blkLocations[startIndex].getCachedHosts() };
private static void checkBlockLocation(final BlockLocation loc, String[] names, String[] hosts, String[] cachedHosts, String[] topologyPaths, final long offset, final long length, final boolean corrupt) throws Exception { assertNotNull(loc.getHosts()); assertNotNull(loc.getCachedHosts()); assertNotNull(loc.getNames()); assertNotNull(loc.getTopologyPaths()); assertArrayEquals(hosts, loc.getHosts()); assertArrayEquals(cachedHosts, loc.getCachedHosts()); assertArrayEquals(names, loc.getNames()); assertArrayEquals(topologyPaths, loc.getTopologyPaths()); assertEquals(offset, loc.getOffset()); assertEquals(length, loc.getLength()); assertEquals(corrupt, loc.isCorrupt()); }
private static void checkBlockLocation(final BlockLocation loc, String[] names, String[] hosts, String[] cachedHosts, String[] topologyPaths, final long offset, final long length, final boolean corrupt) throws Exception { assertNotNull(loc.getHosts()); assertNotNull(loc.getCachedHosts()); assertNotNull(loc.getNames()); assertNotNull(loc.getTopologyPaths()); assertArrayEquals(hosts, loc.getHosts()); assertArrayEquals(cachedHosts, loc.getCachedHosts()); assertArrayEquals(names, loc.getNames()); assertArrayEquals(topologyPaths, loc.getTopologyPaths()); assertEquals(offset, loc.getOffset()); assertEquals(length, loc.getLength()); assertEquals(corrupt, loc.isCorrupt()); }