public static void downloadMostRecentImageToDirectory(URL infoServer, File dir) throws IOException { String fileId = ImageServlet.getParamStringForMostRecentImage(); getFileClient(infoServer, fileId, Lists.newArrayList(dir), null, false); }
@Override public Boolean run() throws Exception { dstImage.getStorage().cTime = sig.cTime; // get fsimage if (sig.mostRecentCheckpointTxId == dstImage.getStorage().getMostRecentCheckpointTxId()) { LOG.info("Image has not changed. Will not download image."); } else { LOG.info("Image has changed. Downloading updated image from NN."); MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage( nnHostPort, sig.mostRecentCheckpointTxId, dstImage.getStorage(), true, false); dstImage.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId, downloadedHash); } // get edits file for (RemoteEditLog log : manifest.getLogs()) { TransferFsImage.downloadEditsToStorage( nnHostPort, log, dstImage.getStorage()); } // true if we haven't loaded all the transactions represented by the // downloaded fsimage. return dstImage.getLastAppliedTxId() < sig.mostRecentCheckpointTxId; } });
/** * Requests that the NameNode download an image from this node. * * @param fsName the http address for the remote NN * @param conf Configuration * @param storage the storage directory to transfer the image from * @param nnf the NameNodeFile type of the image * @param txid the transaction ID of the image to be uploaded * @throws IOException if there is an I/O error */ static TransferResult uploadImageFromStorage(URL fsName, Configuration conf, NNStorage storage, NameNodeFile nnf, long txid) throws IOException { return uploadImageFromStorage(fsName, conf, storage, nnf, txid, null); }
setTimeout(connection); writeFileToPutRequest(conf, connection, imageFile, canceler);
ServletContext context = getServletContext(); FSImage nnImage = (FSImage)context.getAttribute("name.system.image"); TransferFsImage ff = new TransferFsImage(pmap, request, response); if (ff.getImage()) { TransferFsImage.getFileServer(response.getOutputStream(), nnImage.getFsImageName()); } else if (ff.getEdit()) { TransferFsImage.getFileServer(response.getOutputStream(), nnImage.getFsEditName()); } else if (ff.putImage()) { nnImage.validateCheckpointUpload(ff.getToken()); TransferFsImage.getFileClient(ff.getInfoServer(), "getimage=1", nnImage.getFsImageNameCheckpoint()); nnImage.checkpointUploadDone();
LOG.info("Unable to roll forward using only logs. Downloading " + "image with txid " + sig.mostRecentCheckpointTxId); MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage( backupNode.nnHttpAddress, sig.mostRecentCheckpointTxId, bnStorage, true, false); TransferFsImage.downloadEditsToStorage( backupNode.nnHttpAddress, log, bnStorage); TransferFsImage.uploadImageFromStorage(backupNode.nnHttpAddress, conf, bnStorage, NameNodeFile.IMAGE, txid);
setTimeout(connection); "by the namenode when trying to fetch " + url); MD5Hash advertisedDigest = parseMD5Header(connection); String fsImageName = connection .getHeaderField(ImageServlet.HADOOP_IMAGE_EDITS_HEADER); InputStream stream = connection.getInputStream(); return receiveFile(url.toExternalForm(), localPaths, dstStorage, getChecksum, advertisedSize, advertisedDigest, fsImageName, stream, null);
/** * A server-side method to respond to a getfile http request * Copies the contents of the local file into the output stream. */ public static void copyFileToStream(OutputStream out, File localfile, FileInputStream infile, DataTransferThrottler throttler) throws IOException { copyFileToStream(out, localfile, infile, throttler, null); }
/** * Client-side Method to fetch file from a server * Copies the response from the URL to a list of local files. * @param dstStorage if an error occurs writing to one of the files, * this storage object will be notified. * @return a digest of the received file if getChecksum is true */ static MD5Hash getFileClient(URL infoServer, String queryString, List<File> localPaths, Storage dstStorage, boolean getChecksum) throws IOException { URL url = new URL(infoServer, ImageServlet.PATH_SPEC + "?" + queryString); LOG.info("Opening connection to " + url); return doGetUrl(url, localPaths, dstStorage, getChecksum); }
static MD5Hash handleUploadImageRequest(HttpServletRequest request, long imageTxId, Storage dstStorage, InputStream stream, long advertisedSize, DataTransferThrottler throttler) throws IOException { String fileName = NNStorage.getCheckpointImageFileName(imageTxId); List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName); if (dstFiles.isEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash advertisedDigest = parseMD5Header(request); MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true, advertisedSize, advertisedDigest, fileName, stream, throttler); LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + dstFiles.get(0).length() + " bytes."); return hash; }
MD5Hash hash = TransferFsImage.downloadImageToStorage( proxyInfo.getHttpAddress(), imageTxId, storage, true, true); image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId,
@Override public Void run() throws Exception { TransferFsImage.downloadMostRecentImageToDirectory(infoServer, new File(argv[idx])); return null; } });
long start = monotonicNow(); MD5Hash downloadImageDigest = TransferFsImage .handleUploadImageRequest(request, txid, nnImage.getStorage(), stream, parsedParams.getFileSize(), getThrottler(conf));
static MD5Hash handleUploadImageRequest(HttpServletRequest request, long imageTxId, Storage dstStorage, InputStream stream, long advertisedSize, DataTransferThrottler throttler) throws IOException { String fileName = NNStorage.getCheckpointImageFileName(imageTxId); List<File> dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName); if (dstFiles.isEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash advertisedDigest = parseMD5Header(request); MD5Hash hash = Util.receiveFile(fileName, dstFiles, dstStorage, true, advertisedSize, advertisedDigest, fileName, stream, throttler); LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + dstFiles.get(0).length() + " bytes."); return hash; }
ServletContext context = getServletContext(); FSImage nnImage = (FSImage)context.getAttribute("name.system.image"); TransferFsImage ff = new TransferFsImage(pmap, request, response); if (ff.getImage()) { response.setHeader(TransferFsImage.CONTENT_LENGTH, String.valueOf(nnImage.getFsImageName().length())); TransferFsImage.getFileServer(response.getOutputStream(), nnImage.getFsImageName(), nnImage.imageTransferThrottler); } else if (ff.getEdit()) { response.setHeader(TransferFsImage.CONTENT_LENGTH, String.valueOf(nnImage.getFsEditName().length())); TransferFsImage.getFileServer(response.getOutputStream(), nnImage.getFsEditName(), nnImage.imageTransferThrottler); } else if (ff.getEditNew()) { response.setHeader(TransferFsImage.CONTENT_LENGTH, String.valueOf(nnImage.getFsEditNewName().length())); TransferFsImage.getFileServer(response.getOutputStream(), nnImage.getFsEditNewName(), nnImage.imageTransferThrottler); } else if (ff.putImage()) { nnImage.validateCheckpointUpload(ff.getToken()); nnImage.checkpointUploadDone( TransferFsImage.getFileClient(ff.getInfoServer(), "getimage=1",
TransferFsImage.downloadImageToStorage(fsName, 0, dstImage, false); fail("Storage info was not verified"); } catch (IOException ioe) { TransferFsImage.downloadEditsToStorage(fsName, log, dstImage); fail("Storage info was not verified"); } catch (IOException ioe) { TransferFsImage.uploadImageFromStorage(fsName, conf, dstImage, NameNodeFile.IMAGE, 0); fail("Storage info was not verified");
setTimeout(connection); "by the namenode when trying to fetch " + url); MD5Hash advertisedDigest = parseMD5Header(connection); String fsImageName = connection .getHeaderField(ImageServlet.HADOOP_IMAGE_EDITS_HEADER); InputStream stream = connection.getInputStream(); return receiveFile(url.toExternalForm(), localPaths, dstStorage, getChecksum, advertisedSize, advertisedDigest, fsImageName, stream, null);
private static void writeFileToPutRequest(Configuration conf, HttpURLConnection connection, File imageFile, Canceler canceler) throws IOException { connection.setRequestProperty(Util.CONTENT_TYPE, "application/octet-stream"); connection.setRequestProperty(Util.CONTENT_TRANSFER_ENCODING, "binary"); OutputStream output = connection.getOutputStream(); FileInputStream input = new FileInputStream(imageFile); try { copyFileToStream(output, imageFile, input, ImageServlet.getThrottler(conf), canceler); } finally { IOUtils.closeStream(input); IOUtils.closeStream(output); } }
setTimeout(connection); writeFileToPutRequest(conf, connection, imageFile, canceler);
@Override public Void run() throws IOException { // We may have lost our ticket since last checkpoint, log in again, just in case if (UserGroupInformation.isSecurityEnabled()) { UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab(); } boolean success = false; try { TransferFsImage.doGetUrl(url, localPaths, storage, true); assert tmpFile.exists(); success = true; } finally { if (!success) { if (!tmpFile.delete()) { LOG.warn("Failed to delete temporary file " + tmpFile); } } } return null; } });