public LocatedBlocks getLocatedBlocks(String src, long start) throws IOException { return getLocatedBlocks(src, start, dfsClientConf.prefetchSize); }
public LocatedBlocks getLocatedBlocks(String src, long start) throws IOException { return getLocatedBlocks(src, start, dfsClientConf.prefetchSize); }
public LocatedBlocks getLocatedBlocks(Path filePath, long start, long len) throws IOException { if (filePath == null) { return null; } return dfs.getLocatedBlocks(getPathName(filePath), start, len); }
public LocatedBlocks getLocatedBlocks(String src, long start) throws IOException { return getLocatedBlocks(src, start, dfsClientConf.getPrefetchSize()); }
static LocatedBlocks getLocatedBlocks(Path src, DistributedFileSystem dfs) throws IOException { return dfs.getClient().getLocatedBlocks(src.toString(), 0, Long.MAX_VALUE); }
/** * Create an input stream that obtains a nodelist from the * namenode, and then reads from all the right places. Creates * inner subclass of InputStream that does the right out-of-band * work. */ public DFSInputStream open(String src, int buffersize, boolean verifyChecksum) throws IOException { checkOpen(); // Get block info from namenode try (TraceScope ignored = newPathTraceScope("newDFSInputStream", src)) { LocatedBlocks locatedBlocks = getLocatedBlocks(src, 0); return openInternal(locatedBlocks, src, verifyChecksum); } }
/** Fetch a block from namenode and cache it */ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache) throws IOException { synchronized(infoLock) { int targetBlockIdx = locatedBlocks.findBlock(offset); if (targetBlockIdx < 0) { // block is not cached targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx); useCache = false; } if (!useCache) { // fetch blocks final LocatedBlocks newBlocks = (length == 0) ? dfsClient.getLocatedBlocks(src, offset) : dfsClient.getLocatedBlocks(src, offset, length); if (newBlocks == null || newBlocks.locatedBlockCount() == 0) { throw new EOFException("Could not find target position " + offset); } locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks()); } return locatedBlocks.get(targetBlockIdx); } }
/** Fetch a block from namenode and cache it */ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache) throws IOException { synchronized(infoLock) { int targetBlockIdx = locatedBlocks.findBlock(offset); if (targetBlockIdx < 0) { // block is not cached targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx); useCache = false; } if (!useCache) { // fetch blocks final LocatedBlocks newBlocks = (length == 0) ? dfsClient.getLocatedBlocks(src, offset) : dfsClient.getLocatedBlocks(src, offset, length); if (newBlocks == null || newBlocks.locatedBlockCount() == 0) { throw new EOFException("Could not find target position " + offset); } locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks()); } return locatedBlocks.get(targetBlockIdx); } }
/** Fetch a block from namenode and cache it */ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache) throws IOException { synchronized(infoLock) { int targetBlockIdx = locatedBlocks.findBlock(offset); if (targetBlockIdx < 0) { // block is not cached targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx); useCache = false; } if (!useCache) { // fetch blocks final LocatedBlocks newBlocks = (length == 0) ? dfsClient.getLocatedBlocks(src, offset) : dfsClient.getLocatedBlocks(src, offset, length); if (newBlocks == null || newBlocks.locatedBlockCount() == 0) { throw new EOFException("Could not find target position " + offset); } locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks()); } return locatedBlocks.get(targetBlockIdx); } }
@Override public Boolean get() { try { LocatedBlocks lbs = dfsClient.getLocatedBlocks(p.toString(), 0); LocatedBlock lb = lbs.get(0); List<DatanodeInfo> locations = Arrays.asList(lb.getLocations()); DFSClient.LOG .info("Source :" + source + ", destination: " + destination); DFSClient.LOG.info("Got updated locations :" + locations); return locations.contains(destination) && !locations.contains(source); } catch (IOException e) { DFSClient.LOG.error("Problem in getting block locations", e); } return null; } }, 1000, 10000);
/** create a file with a length of <code>fileLen</code> */ private List<LocatedBlock> createFile(String fileNamePrefix, long fileLen, boolean isLazyPersist) throws IOException { FileSystem fs = cluster.getFileSystem(); Path filePath = new Path("/" + fileNamePrefix + ".dat"); DFSTestUtil.createFile( fs, filePath, isLazyPersist, 1024, fileLen, BLOCK_LENGTH, (short) 1, r.nextLong(), false); return client.getLocatedBlocks(filePath.toString(), 0, fileLen).getLocatedBlocks(); }
private LocatedBlock getLocatedBlock() throws IOException { LocatedBlocks locatedBlocks = client.getLocatedBlocks(PATH.toString(), 0, BLOCK_SIZE); assertThat(locatedBlocks.getLocatedBlocks().size(), is(1)); return Iterables.getOnlyElement(locatedBlocks.getLocatedBlocks()); }
private FileEncryptionInfo getFileEncryptionInfo(Path path) throws Exception { LocatedBlocks blocks = fs.getClient().getLocatedBlocks(path.toString(), 0); return blocks.getFileEncryptionInfo(); }
private void waitForAllReplicas(int expectedReplicaNum, Path file, DistributedFileSystem dfs) throws Exception { for (int i = 0; i < 5; i++) { LocatedBlocks lbs = dfs.getClient().getLocatedBlocks(file.toString(), 0, BLOCK_SIZE); LocatedBlock lb = lbs.get(0); if (lb.getLocations().length >= expectedReplicaNum) { return; } else { Thread.sleep(1000); } } }
private LocatedBlocks createFileGetBlocks(String filenamePrefix) throws IOException { Path filePath = new Path("/" + filenamePrefix + ".dat"); // Write out a file with a few blocks, get block locations. DFSTestUtil.createFile(fs, filePath, BLOCK_SIZE, BLOCK_SIZE * NUM_BLOCKS, BLOCK_SIZE, NUM_DATANODES, seed); // Get the block list for the file with the block locations. LocatedBlocks blocks = client.getLocatedBlocks( filePath.toString(), 0, BLOCK_SIZE * NUM_BLOCKS); assertThat(cluster.getNamesystem().getUnderReplicatedBlocks(), is(0L)); return blocks; }
protected final LocatedBlocks ensureFileReplicasOnStorageType( Path path, StorageType storageType) throws IOException { // Ensure that returned block locations returned are correct! LOG.info("Ensure path: " + path + " is on StorageType: " + storageType); assertThat(fs.exists(path), is(true)); long fileLength = client.getFileInfo(path.toString()).getLen(); LocatedBlocks locatedBlocks = client.getLocatedBlocks(path.toString(), 0, fileLength); for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) { assertThat(locatedBlock.getStorageTypes()[0], is(storageType)); } return locatedBlocks; }
/** * Get block location info about file * * getBlockLocations() returns a list of hostnames that store * data for a specific file region. It returns a set of hostnames * for every block within the indicated region. * * This function is very useful when writing code that considers * data-placement when performing operations. For example, the * MapReduce system tries to schedule tasks on the same machines * as the data-block the task processes. */ public BlockLocation[] getBlockLocations(String src, long start, long length) throws IOException, UnresolvedLinkException { TraceScope scope = getPathTraceScope("getBlockLocations", src); try { LocatedBlocks blocks = getLocatedBlocks(src, start, length); BlockLocation[] locations = DFSUtil.locatedBlocks2Locations(blocks); HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length]; for (int i = 0; i < locations.length; i++) { hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i)); } return hdfsLocations; } finally { scope.close(); } }
/** * Get block location info about file * * getBlockLocations() returns a list of hostnames that store * data for a specific file region. It returns a set of hostnames * for every block within the indicated region. * * This function is very useful when writing code that considers * data-placement when performing operations. For example, the * MapReduce system tries to schedule tasks on the same machines * as the data-block the task processes. */ public BlockLocation[] getBlockLocations(String src, long start, long length) throws IOException, UnresolvedLinkException { TraceScope scope = getPathTraceScope("getBlockLocations", src); try { LocatedBlocks blocks = getLocatedBlocks(src, start, length); BlockLocation[] locations = DFSUtil.locatedBlocks2Locations(blocks); HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length]; for (int i = 0; i < locations.length; i++) { hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i)); } return hdfsLocations; } finally { scope.close(); } }
private Replication getOrVerifyReplication(Path file, Replication expected) throws IOException { final List<LocatedBlock> lbs = dfs.getClient().getLocatedBlocks( file.toString(), 0).getLocatedBlocks(); Assert.assertEquals(1, lbs.size()); LocatedBlock lb = lbs.get(0); StringBuilder types = new StringBuilder(); final Replication r = new Replication(); for(StorageType t : lb.getStorageTypes()) { types.append(t).append(", "); if (t == StorageType.DISK) { r.disk++; } else if (t == StorageType.ARCHIVE) { r.archive++; } else { Assert.fail("Unexpected storage type " + t); } } if (expected != null) { final String s = "file = " + file + "\n types = [" + types + "]"; Assert.assertEquals(s, expected, r); } return r; } }
@Override void prepare() throws Exception { final Path filePath = new Path(file); DFSTestUtil.createFile(dfs, filePath, BlockSize, DataNodes, 0); // append to the file and leave the last block under construction out = this.client.append(file, BlockSize, EnumSet.of(CreateFlag.APPEND), null, null); byte[] appendContent = new byte[100]; new Random().nextBytes(appendContent); out.write(appendContent); ((HdfsDataOutputStream) out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH)); LocatedBlocks blks = dfs.getClient() .getLocatedBlocks(file, BlockSize + 1); assertEquals(1, blks.getLocatedBlocks().size()); nodes = blks.get(0).getLocations(); oldBlock = blks.get(0).getBlock(); LocatedBlock newLbk = client.getNamenode().updateBlockForPipeline( oldBlock, client.getClientName()); newBlock = new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(), oldBlock.getNumBytes(), newLbk.getBlock().getGenerationStamp()); }