/** * Create a compression instance based on the user's configuration in the given * Configuration object. * @throws IOException if the specified codec is not available. */ static FSImageCompression createCompression(Configuration conf) throws IOException { boolean compressImage = conf.getBoolean( DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESS_DEFAULT); if (!compressImage) { return createNoopCompression(); } String codecClassName = conf.get( DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT); return createCompression(conf, codecClassName); }
public static InputStream wrapInputStreamForCompression( Configuration conf, String codec, InputStream in) throws IOException { if (codec.isEmpty()) return in; FSImageCompression compression = FSImageCompression.createCompression( conf, codec); CompressionCodec imageCodec = compression.getImageCodec(); return imageCodec.createInputStream(in); }
/** * Create a compression instance based on a header read from an input stream. * @throws IOException if the specified codec is not available or the * underlying IO fails. */ static FSImageCompression readCompressionHeader( Configuration conf, DataInput in) throws IOException { boolean isCompressed = in.readBoolean(); if (!isCompressed) { return createNoopCompression(); } else { String codecClassName = Text.readString(in); return createCompression(conf, codecClassName); } }
/** * Save FSimage in the legacy format. This is not for NN consumption, * but for tools like OIV. */ public void saveLegacyOIVImage(FSNamesystem source, String targetDir, Canceler canceler) throws IOException { FSImageCompression compression = FSImageCompression.createCompression(conf); long txid = getCorrectLastAppliedOrWrittenTxId(); SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler); FSImageFormat.Saver saver = new FSImageFormat.Saver(ctx); String imageFileName = NNStorage.getLegacyOIVImageFileName(txid); File imageFile = new File(targetDir, imageFileName); saver.save(imageFile, compression); archivalManager.purgeOldLegacyOIVImages(targetDir, txid); }
/** * Save the contents of the FS image to the file. */ void saveFSImage(SaveNamespaceContext context, StorageDirectory sd, NameNodeFile dstType) throws IOException { long txid = context.getTxId(); File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid); File dstFile = NNStorage.getStorageFile(sd, dstType, txid); FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); long numErrors = saver.save(newFile, compression); if (numErrors > 0) { // The image is likely corrupted. LOG.error("Detected " + numErrors + " errors while saving FsImage " + dstFile); exitAfterSave.set(true); } MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest()); storage.setMostRecentCheckpointInfo(txid, Time.now()); }
public Options codec(String codec) throws IOException { this.compress = FSImageCompression.createCompression(getConf(), codec); return this; }
/** * Create a compression instance based on the user's configuration in the given * Configuration object. * @throws IOException if the specified codec is not available. */ static FSImageCompression createCompression(Configuration conf) throws IOException { boolean compressImage = conf.getBoolean( DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESS_DEFAULT); if (!compressImage) { return createNoopCompression(); } String codecClassName = conf.get( DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT); return createCompression(conf, codecClassName); }
/** * Create a compression instance based on the user's configuration in the given * Configuration object. * @throws IOException if the specified codec is not available. */ static FSImageCompression createCompression(Configuration conf, boolean forceUncompressed) throws IOException { boolean compressImage = (!forceUncompressed) && conf.getBoolean( HdfsConstants.DFS_IMAGE_COMPRESS_KEY, HdfsConstants.DFS_IMAGE_COMPRESS_DEFAULT); if (!compressImage) { return createNoopCompression(); } String codecClassName = conf.get( HdfsConstants.DFS_IMAGE_COMPRESSION_CODEC_KEY, HdfsConstants.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT); return createCompression(conf, codecClassName); }
/** * Create a compression instance based on the user's configuration in the given * Configuration object. * @throws IOException if the specified codec is not available. */ static FSImageCompression createCompression(Configuration conf) throws IOException { boolean compressImage = conf.getBoolean( DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESS_DEFAULT); if (!compressImage) { return createNoopCompression(); } String codecClassName = conf.get( DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT); return createCompression(conf, codecClassName); }
public static InputStream wrapInputStreamForCompression( Configuration conf, String codec, InputStream in) throws IOException { if (codec.isEmpty()) return in; FSImageCompression compression = FSImageCompression.createCompression( conf, codec); CompressionCodec imageCodec = compression.getImageCodec(); return imageCodec.createInputStream(in); }
public static InputStream wrapInputStreamForCompression( Configuration conf, String codec, InputStream in) throws IOException { if (codec.isEmpty()) return in; FSImageCompression compression = FSImageCompression.createCompression( conf, codec); CompressionCodec imageCodec = compression.getImageCodec(); return imageCodec.createInputStream(in); }
/** * Create a compression instance based on a header read from an input stream. * @throws IOException if the specified codec is not available or the * underlying IO fails. */ static FSImageCompression readCompressionHeader( Configuration conf, DataInput in) throws IOException { boolean isCompressed = in.readBoolean(); if (!isCompressed) { return createNoopCompression(); } else { String codecClassName = Text.readString(in); return createCompression(conf, codecClassName); } }
/** * Create a compression instance based on a header read from an input stream. * @throws IOException if the specified codec is not available or the * underlying IO fails. */ static FSImageCompression readCompressionHeader( Configuration conf, DataInput in) throws IOException { boolean isCompressed = in.readBoolean(); if (!isCompressed) { return createNoopCompression(); } else { String codecClassName = Text.readString(in); return createCompression(conf, codecClassName); } }
/** * Create a compression instance based on a header read from an input stream. * @throws IOException if the specified codec is not available or the * underlying IO fails. */ static FSImageCompression readCompressionHeader( Configuration conf, DataInputStream dis) throws IOException { boolean isCompressed = dis.readBoolean(); if (!isCompressed) { return createNoopCompression(); } else { String codecClassName = Text.readString(dis); return createCompression(conf, codecClassName); } }
void saveFSImage(String dest, DataOutputStream fstream) throws IOException { saveNamespaceContext.set(namesystem, editLog.getLastWrittenTxId()); FSImageFormat.Saver saver = new FSImageFormat.Saver(saveNamespaceContext); FSImageCompression compression = FSImageCompression.createCompression(namesystem.getConf(), false); saver.save(new File(dest), compression, fstream); }
/** * Save FSimage in the legacy format. This is not for NN consumption, * but for tools like OIV. */ public void saveLegacyOIVImage(FSNamesystem source, String targetDir, Canceler canceler) throws IOException { FSImageCompression compression = FSImageCompression.createCompression(conf); long txid = getCorrectLastAppliedOrWrittenTxId(); SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler); FSImageFormat.Saver saver = new FSImageFormat.Saver(ctx); String imageFileName = NNStorage.getLegacyOIVImageFileName(txid); File imageFile = new File(targetDir, imageFileName); saver.save(imageFile, compression); archivalManager.purgeOldLegacyOIVImages(targetDir, txid); }
/** * Save FSimage in the legacy format. This is not for NN consumption, * but for tools like OIV. */ public void saveLegacyOIVImage(FSNamesystem source, String targetDir, Canceler canceler) throws IOException { FSImageCompression compression = FSImageCompression.createCompression(conf); long txid = getCorrectLastAppliedOrWrittenTxId(); SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler); FSImageFormat.Saver saver = new FSImageFormat.Saver(ctx); String imageFileName = NNStorage.getLegacyOIVImageFileName(txid); File imageFile = new File(targetDir, imageFileName); saver.save(imageFile, compression); archivalManager.purgeOldLegacyOIVImages(targetDir, txid); }
/** * Save the contents of the FS image to the file. */ void saveFSImage(SaveNamespaceContext context, StorageDirectory sd, NameNodeFile dstType) throws IOException { long txid = context.getTxId(); File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid); File dstFile = NNStorage.getStorageFile(sd, dstType, txid); FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); saver.save(newFile, compression); MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest()); storage.setMostRecentCheckpointInfo(txid, Time.now()); }
/** * Save the contents of the FS image to the file. */ void saveFSImage(SaveNamespaceContext context, StorageDirectory sd, NameNodeFile dstType) throws IOException { long txid = context.getTxId(); File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid); File dstFile = NNStorage.getStorageFile(sd, dstType, txid); FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); saver.save(newFile, compression); MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest()); storage.setMostRecentCheckpointInfo(txid, Time.now()); }
/** Save the fsimage to a temp file */ private File saveFSImageToTempFile() throws IOException { SaveNamespaceContext context = new SaveNamespaceContext(fsn, txid, new Canceler()); FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); File imageFile = getImageFile(testDir, txid); fsn.readLock(); try { saver.save(imageFile, compression); } finally { fsn.readUnlock(); } return imageFile; }