public static void downloadMostRecentImageToDirectory(URL infoServer, File dir) throws IOException { String fileId = ImageServlet.getParamStringForMostRecentImage(); getFileClient(infoServer, fileId, Lists.newArrayList(dir), null, false); }
public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId, Storage dstStorage, boolean needDigest, boolean isBootstrapStandby) throws IOException { String fileid = ImageServlet.getParamStringForImage(null, imageTxId, dstStorage, isBootstrapStandby); String fileName = NNStorage.getCheckpointImageFileName(imageTxId); List<File> dstFiles = dstStorage.getFiles( NameNodeDirType.IMAGE, fileName); if (dstFiles.isEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest); LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + dstFiles.get(0).length() + " bytes."); return hash; }
List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, tmpFileName); getFileClient(fsName, fileid, tmpFiles, dstStorage, false); LOG.info("Downloaded file " + tmpFiles.get(0).getName() + " size " + finalFiles.get(0).length() + " bytes.");
/** * Copy the new fsimage into the NameNode */ private void putFSImage(CheckpointSignature sig) throws IOException { String fileid = "putimage=1&port=" + infoPort + "&machine=" + InetAddress.getLocalHost().getHostAddress() + "&token=" + sig.toString(); LOG.info("Posted URL " + fsName + fileid); TransferFsImage.getFileClient(fsName, fileid, (File[])null); }
/** * Copy the new fsimage into the NameNode */ private void putFSImage(CheckpointSignature sig) throws IOException { String fileid = "putimage=1&port=" + infoPort + "&machine=" + InetAddress.getLocalHost().getHostAddress() + "&token=" + sig.toString(); LOG.info("Posted URL " + fsName + fileid); TransferFsImage.getFileClient(fsName, fileid, (File[])null, false); }
public static void downloadMostRecentImageToDirectory(URL infoServer, File dir) throws IOException { String fileId = ImageServlet.getParamStringForMostRecentImage(); getFileClient(infoServer, fileId, Lists.newArrayList(dir), null, false); }
public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId, Storage dstStorage, boolean needDigest) throws IOException { String fileid = ImageServlet.getParamStringForImage(null, imageTxId, dstStorage); String fileName = NNStorage.getCheckpointImageFileName(imageTxId); List<File> dstFiles = dstStorage.getFiles( NameNodeDirType.IMAGE, fileName); if (dstFiles.isEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest); LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + dstFiles.get(0).length() + " bytes."); return hash; }
public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId, Storage dstStorage, boolean needDigest) throws IOException { String fileid = ImageServlet.getParamStringForImage(null, imageTxId, dstStorage); String fileName = NNStorage.getCheckpointImageFileName(imageTxId); List<File> dstFiles = dstStorage.getFiles( NameNodeDirType.IMAGE, fileName); if (dstFiles.isEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest); LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + dstFiles.get(0).length() + " bytes."); return hash; }
public static void downloadMostRecentImageToDirectory(URL infoServer, File dir) throws IOException { String fileId = ImageServlet.getParamStringForMostRecentImage(); getFileClient(infoServer, fileId, Lists.newArrayList(dir), null, false); }
/** * Download <code>fsimage</code> and <code>edits</code> * files from the name-node. * @throws IOException */ private void downloadCheckpointFiles(CheckpointSignature sig ) throws IOException { checkpointImage.cTime = sig.cTime; checkpointImage.checkpointTime = sig.checkpointTime; // get fsimage String fileid = "getimage=1"; File[] srcNames = checkpointImage.getImageFiles(); assert srcNames.length > 0 : "No checkpoint targets."; TransferFsImage.getFileClient(fsName, fileid, srcNames); LOG.info("Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes."); // get edits file fileid = "getedit=1"; srcNames = checkpointImage.getEditsFiles(); assert srcNames.length > 0 : "No checkpoint targets."; TransferFsImage.getFileClient(fsName, fileid, srcNames); LOG.info("Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes."); checkpointImage.checkpointUploadDone(); }
assert srcNames.length > 0 : "No checkpoint targets."; fileid = "getimage=1"; TransferFsImage.getFileClient(fsName, fileid, srcNames, false); checkpointImage.imageDigest = sig.imageDigest; LOG.info("Downloaded file " + srcNames[0].getName() + " size " + srcNames = checkpointImage.getEditsFiles(); assert srcNames.length > 0 : "No checkpoint targets."; TransferFsImage.getFileClient(fsName, fileid, srcNames, false); LOG.info("Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes.");
TransferFsImage.getFileClient(ff.getInfoServer(), "getimage=1", nnImage.getFsImageNameCheckpoint(), true));
TransferFsImage.getFileClient(ff.getInfoServer(), "getimage=1", nnImage.getFsImageNameCheckpoint()); nnImage.checkpointUploadDone();
List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, tmpFileName); getFileClient(fsName, fileid, tmpFiles, dstStorage, false); LOG.info("Downloaded file " + tmpFiles.get(0).getName() + " size " + finalFiles.get(0).length() + " bytes.");
/** * Similar to the above test, except that there are multiple local files * and one of them can be saved. */ @Test public void testClientSideExceptionOnJustOneDir() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(0).build(); NNStorage mockStorage = Mockito.mock(NNStorage.class); List<File> localPaths = ImmutableList.of( new File("/xxxxx-does-not-exist/blah"), new File(TEST_DIR, "testfile") ); try { URL fsName = DFSUtil.getInfoServer( cluster.getNameNode().getServiceRpcAddress(), conf, DFSUtil.getHttpClientScheme(conf)).toURL(); String id = "getimage=1&txid=0"; TransferFsImage.getFileClient(fsName, id, localPaths, mockStorage, false); Mockito.verify(mockStorage).reportErrorOnFile(localPaths.get(0)); assertTrue("The valid local file should get saved properly", localPaths.get(1).length() > 0); } finally { cluster.shutdown(); } }
List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, tmpFileName); getFileClient(fsName, fileid, tmpFiles, dstStorage, false); LOG.info("Downloaded file " + tmpFiles.get(0).getName() + " size " + finalFiles.get(0).length() + " bytes.");
/** * Regression test for HDFS-1997. Test that, if an exception * occurs on the client side, it is properly reported as such */ @Test public void testClientSideException() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(0).build(); try { String fsName = NameNode.getHostPortString( cluster.getNameNode().getHttpAddress()); String id = "getimage=1"; File[] localPath = new File[] { new File("/xxxxx-does-not-exist/blah") }; TransferFsImage.getFileClient(fsName, id, localPath); fail("Didn't get an exception!"); } catch (IOException ioe) { assertTrue("Expected FNFE, got: " + StringUtils.stringifyException(ioe), ioe instanceof FileNotFoundException); } finally { cluster.shutdown(); } } }
String id = "getimage=1&txid=0"; TransferFsImage.getFileClient(fsName, id, localPath, mockStorage, false); fail("Didn't get an exception!"); } catch (IOException ioe) {
/** * Test to verify the read timeout */ @Test(timeout = 5000) public void testGetImageTimeout() throws Exception { HttpServer2 testServer = HttpServerFunctionalTest.createServer("hdfs"); try { testServer.addServlet("ImageTransfer", ImageServlet.PATH_SPEC, TestImageTransferServlet.class); testServer.start(); URL serverURL = HttpServerFunctionalTest.getServerURL(testServer); TransferFsImage.timeout = 2000; try { TransferFsImage.getFileClient(serverURL, "txid=1", null, null, false); fail("TransferImage Should fail with timeout"); } catch (SocketTimeoutException e) { assertEquals("Read should timeout", "Read timed out", e.getMessage()); } } finally { if (testServer != null) { testServer.stop(); } } }