/** * Convert IGFS affinity block location into Hadoop affinity block location. * * @param block IGFS affinity block location. * @return Hadoop affinity block location. */ private IgfsBlockLocation convertBlockLocation(BlockLocation block) { try { String[] names = block.getNames(); String[] hosts = block.getHosts(); return new IgfsBlockLocationImpl( block.getOffset(), block.getLength(), Arrays.asList(names), Arrays.asList(hosts)); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed convert block location: " + block); } }
assertEquals(1, locations.length); Collections.sort(expectedWorkerNames, (x, y) -> x.toString().compareTo(y.toString())); String[] actualNames = locations[0].getNames(); String[] actualHosts = locations[0].getHosts(); Arrays.sort(actualNames);
lbs = dfs.getFileBlockLocations(f, 0, 1); } while (lbs.length != 1 && lbs[0].getLength() != repCount); final String name = lbs[0].getNames()[0]; Assert.assertTrue(name.indexOf(':') > 0); String portS = name.substring(name.indexOf(':') + 1);
for (BlockLocation lbk : lbks) { locations: for (String info : lbk.getNames()) { for (int j = 0; j < FAVORED_NODES_NUM; j++) { if (info.equals(nodeNames[(i + j) % REGION_SERVERS])) {
private static Map<String, Object> toJsonMap( final BlockLocation blockLocation) throws IOException { if (blockLocation == null) { return null; } final Map<String, Object> m = new HashMap<>(); m.put("length", blockLocation.getLength()); m.put("offset", blockLocation.getOffset()); m.put("corrupt", blockLocation.isCorrupt()); m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes())); m.put("cachedHosts", blockLocation.getCachedHosts()); m.put("hosts", blockLocation.getHosts()); m.put("names", blockLocation.getNames()); m.put("topologyPaths", blockLocation.getTopologyPaths()); return m; }
@Override public List<String> getFileLocations(String path, long offset) throws IOException { List<String> ret = new ArrayList<String>(); try { FileStatus fStatus = mFs.getFileStatus(new Path(path)); BlockLocation[] bLocations = mFs.getFileBlockLocations(fStatus, offset, 1); if (bLocations.length > 0) { String[] names = bLocations[0].getNames(); Collections.addAll(ret, names); } } catch (IOException e) { LOG.error("Unable to get file location for " + path, e); } return ret; }
/** Return the number of replicas for a given block in the file. */ private static int getNumReplicas(FileSystem fs, Path file, int blockIdx) throws IOException { BlockLocation locs[] = fs.getFileBlockLocations(file, 0, Long.MAX_VALUE); return locs.length < blockIdx + 1 ? 0 : locs[blockIdx].getNames().length; }
(((BlockStorageLocation) blockLocation).getVolumeIds()) : null; String[] hosts = blockLocation.getHosts(); String[] names = blockLocation.getNames(); String[] topologyPaths = blockLocation.getTopologyPaths(); for (int i = 0; i < topologyPaths.length; i++) {
public void testDataNodeRedirect() throws Exception { if (hdfs.exists(TEST_FILE)) { hdfs.delete(TEST_FILE, true); } FSDataOutputStream out = hdfs.create(TEST_FILE, (short) 1); out.writeBytes("0123456789"); out.close(); BlockLocation[] locations = hdfs.getFileBlockLocations(TEST_FILE, 0, 10); String locationName = locations[0].getNames()[0]; URL u = hftpFs.getNamenodeFileURL(TEST_FILE); HttpURLConnection conn = (HttpURLConnection)u.openConnection(); conn.setFollowRedirects(true); conn.connect(); conn.getInputStream(); boolean checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port for (DataNode node : cluster.getDataNodes()) { if (node.getDatanodeRegistration().getName().equals(locationName)) { checked = true; assertEquals(node.getDatanodeRegistration().getInfoPort(), conn.getURL().getPort()); } } assertTrue("The test never checked that location of " + "the block and hftp desitnation are the same", checked); } /**
public BlockStorageLocation(BlockLocation loc, VolumeId[] volumeIds) throws IOException { // Initialize with data from passed in BlockLocation super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), loc .getOffset(), loc.getLength(), loc.isCorrupt()); this.volumeIds = volumeIds; }
public BlockStorageLocation(BlockLocation loc, VolumeId[] volumeIds) throws IOException { // Initialize with data from passed in BlockLocation super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), loc .getOffset(), loc.getLength(), loc.isCorrupt()); this.volumeIds = volumeIds; }
String xferAddr = locations[0].getNames()[0];
verifyDeserialization(bl2.getNames(), names); verifyDeserialization(bl2.getTopologyPaths(), topologyPaths); assertEquals(bl2.getOffset(), offset);
@Test(timeout=180000) public void testFavoredNodesEndToEnd() throws Exception { //create 10 files with random preferred nodes for (int i = 0; i < NUM_FILES; i++) { Random rand = new Random(System.currentTimeMillis() + i); //pass a new created rand so as to get a uniform distribution each time //without too much collisions (look at the do-while loop in getDatanodes) InetSocketAddress datanode[] = getDatanodes(rand); Path p = new Path("/filename"+i); FSDataOutputStream out = dfs.create(p, FsPermission.getDefault(), true, 4096, (short)3, 4096L, null, datanode); out.write(SOME_BYTES); out.close(); BlockLocation[] locations = getBlockLocations(p); //verify the files got created in the right nodes for (BlockLocation loc : locations) { String[] hosts = loc.getNames(); String[] hosts1 = getStringForInetSocketAddrs(datanode); assertTrue(compareNodes(hosts, hosts1)); } } }
public void testGetFileBlockLocations() throws IOException { final String f = "/test/testGetFileBlockLocations"; createFile(path(f)); final BlockLocation[] computed = fs.getFileBlockLocations(new Path(f), 0L, 1L); final BlockLocation[] expected = cluster.getFileSystem().getFileBlockLocations( new Path(f), 0L, 1L); assertEquals(expected.length, computed.length); for (int i = 0; i < computed.length; i++) { assertEquals(expected[i].toString(), computed[i].toString()); // Check names String names1[] = expected[i].getNames(); String names2[] = computed[i].getNames(); Arrays.sort(names1); Arrays.sort(names2); Assert.assertArrayEquals("Names differ", names1, names2); // Check topology String topos1[] = expected[i].getTopologyPaths(); String topos2[] = computed[i].getTopologyPaths(); Arrays.sort(topos1); Arrays.sort(topos2); Assert.assertArrayEquals("Topology differs", topos1, topos2); } }
@Test(timeout = 180000) public void testFavoredNodesEndToEndForAppend() throws Exception { // create 10 files with random preferred nodes for (int i = 0; i < NUM_FILES; i++) { Random rand = new Random(System.currentTimeMillis() + i); // pass a new created rand so as to get a uniform distribution each time // without too much collisions (look at the do-while loop in getDatanodes) InetSocketAddress datanode[] = getDatanodes(rand); Path p = new Path("/filename" + i); // create and close the file. dfs.create(p, FsPermission.getDefault(), true, 4096, (short) 3, 4096L, null, null).close(); // re-open for append FSDataOutputStream out = dfs.append(p, EnumSet.of(CreateFlag.APPEND), 4096, null, datanode); out.write(SOME_BYTES); out.close(); BlockLocation[] locations = getBlockLocations(p); // verify the files got created in the right nodes for (BlockLocation loc : locations) { String[] hosts = loc.getNames(); String[] hosts1 = getStringForInetSocketAddrs(datanode); assertTrue(compareNodes(hosts, hosts1)); } } }
@Test public void testGetFileBlockLocations() throws IOException { Path path = new Path("/foo/10.0.0.1@bar"); BlockLocation[] locations = fs.getFileBlockLocations(new FileStatus(1027, false, 1, 4096, 123456, path), 7, 1024); assertEquals(1, locations.length); // BlockLocation has no good equals method BlockLocation location = locations[0]; assertArrayEquals(new String[] { "10.0.0.1:1234", "10.0.0.1:9012" }, location.getNames()); assertArrayEquals(new String[] { "10.0.0.1" }, location.getHosts()); // Block covers the whole file assertEquals(0, location.getOffset()); assertEquals(1027, location.getLength()); }
private static void checkBlockLocation(final BlockLocation loc, String[] names, String[] hosts, String[] cachedHosts, String[] topologyPaths, final long offset, final long length, final boolean corrupt) throws Exception { assertNotNull(loc.getHosts()); assertNotNull(loc.getCachedHosts()); assertNotNull(loc.getNames()); assertNotNull(loc.getTopologyPaths()); assertArrayEquals(hosts, loc.getHosts()); assertArrayEquals(cachedHosts, loc.getCachedHosts()); assertArrayEquals(names, loc.getNames()); assertArrayEquals(topologyPaths, loc.getTopologyPaths()); assertEquals(offset, loc.getOffset()); assertEquals(length, loc.getLength()); assertEquals(corrupt, loc.isCorrupt()); }
private static void checkBlockLocation(final BlockLocation loc, String[] names, String[] hosts, String[] cachedHosts, String[] topologyPaths, final long offset, final long length, final boolean corrupt) throws Exception { assertNotNull(loc.getHosts()); assertNotNull(loc.getCachedHosts()); assertNotNull(loc.getNames()); assertNotNull(loc.getTopologyPaths()); assertArrayEquals(hosts, loc.getHosts()); assertArrayEquals(cachedHosts, loc.getCachedHosts()); assertArrayEquals(names, loc.getNames()); assertArrayEquals(topologyPaths, loc.getTopologyPaths()); assertEquals(offset, loc.getOffset()); assertEquals(length, loc.getLength()); assertEquals(corrupt, loc.isCorrupt()); }