/** * Create a compression instance based on the user's configuration in the given * Configuration object. * @throws IOException if the specified codec is not available. */ static FSImageCompression createCompression(Configuration conf) throws IOException { boolean compressImage = conf.getBoolean( DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESS_DEFAULT); if (!compressImage) { return createNoopCompression(); } String codecClassName = conf.get( DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT); return createCompression(conf, codecClassName); }
/** * Create a "noop" compression - i.e. uncompressed */ static FSImageCompression createNoopCompression() { return new FSImageCompression(); }
public static InputStream wrapInputStreamForCompression( Configuration conf, String codec, InputStream in) throws IOException { if (codec.isEmpty()) return in; FSImageCompression compression = FSImageCompression.createCompression( conf, codec); CompressionCodec imageCodec = compression.getImageCodec(); return imageCodec.createInputStream(in); }
if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.FSIMAGE_COMPRESSION, imgVersion)) { compression = FSImageCompression.readCompressionHeader(conf, in); } else { compression = FSImageCompression.createNoopCompression(); in = compression.unwrapInputStream(fin);
/** * Save FSimage in the legacy format. This is not for NN consumption, * but for tools like OIV. */ public void saveLegacyOIVImage(FSNamesystem source, String targetDir, Canceler canceler) throws IOException { FSImageCompression compression = FSImageCompression.createCompression(conf); long txid = getCorrectLastAppliedOrWrittenTxId(); SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler); FSImageFormat.Saver saver = new FSImageFormat.Saver(ctx); String imageFileName = NNStorage.getLegacyOIVImageFileName(txid); File imageFile = new File(targetDir, imageFileName); saver.save(imageFile, compression); archivalManager.purgeOldLegacyOIVImages(targetDir, txid); }
context.getSourceNamesystem().getEffectiveLayoutVersion()); codec = compression.getImageCodec(); if (codec != null) { b.setCodec(codec.getClass().getCanonicalName());
out = compression.writeHeaderAndWrapStream(fos); LOG.info("Saving image file " + newFile + " using " + compression);
compression = FSImageCompression.readCompressionHeader(conf, in); } else { compression = FSImageCompression.createNoopCompression(); in = compression.unwrapInputStream(in);
/** * Save the contents of the FS image to the file. */ void saveFSImage(SaveNamespaceContext context, StorageDirectory sd, NameNodeFile dstType) throws IOException { long txid = context.getTxId(); File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid); File dstFile = NNStorage.getStorageFile(sd, dstType, txid); FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); long numErrors = saver.save(newFile, compression); if (numErrors > 0) { // The image is likely corrupted. LOG.error("Detected " + numErrors + " errors while saving FsImage " + dstFile); exitAfterSave.set(true); } MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest()); storage.setMostRecentCheckpointInfo(txid, Time.now()); }
OutputStream beginSection(OutputStream out) throws IOException { CompressionCodec codec = compress.getImageCodec(); if (null == codec) { return out; } return codec.createOutputStream(out); }
out = compression.writeHeaderAndWrapStream(out); FSNamesystem.LOG.info("Saving image file " + newFile + " using " + compression);
/** * Create a compression instance based on a header read from an input stream. * @throws IOException if the specified codec is not available or the * underlying IO fails. */ static FSImageCompression readCompressionHeader( Configuration conf, DataInput in) throws IOException { boolean isCompressed = in.readBoolean(); if (!isCompressed) { return createNoopCompression(); } else { String codecClassName = Text.readString(in); return createCompression(conf, codecClassName); } }
if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.FSIMAGE_COMPRESSION, imgVersion)) { compression = FSImageCompression.readCompressionHeader(conf, in); } else { compression = FSImageCompression.createNoopCompression(); in = compression.unwrapInputStream(fin);
public Options codec(String codec) throws IOException { this.compress = FSImageCompression.createCompression(getConf(), codec); return this; }
public static InputStream wrapInputStreamForCompression( Configuration conf, String codec, InputStream in) throws IOException { if (codec.isEmpty()) return in; FSImageCompression compression = FSImageCompression.createCompression( conf, codec); CompressionCodec imageCodec = compression.getImageCodec(); return imageCodec.createInputStream(in); }
/** * Create a compression instance using the codec specified by * <code>codecClassName</code> */ static FSImageCompression createCompression(Configuration conf, String codecClassName) throws IOException { CompressionCodecFactory factory = new CompressionCodecFactory(conf); CompressionCodec codec = factory.getCodecByClassName(codecClassName); if (codec == null) { throw new IOException("Not a supported codec: " + codecClassName); } return new FSImageCompression(codec); }
@Override public synchronized String toString() { StringBuilder sb = new StringBuilder(); sb.append("{ codec=\"").append(compress.getImageCodec()); sb.append("\", startBlock=").append(startBlock); sb.append(", curBlock=").append(curBlock); sb.append(", startInode=").append(startInode); sb.append(", curInode=").append(curInode); sb.append(", ugi=").append(ugis); sb.append(", blockIds=").append(blockIds); sb.append(", offset=").append(raw.pos); sb.append(" }"); return sb.toString(); }
out = compression.writeHeaderAndWrapStream(fos); LOG.info("Saving image file " + newFile + " using " + compression);
/** * Create a compression instance based on the user's configuration in the given * Configuration object. * @throws IOException if the specified codec is not available. */ static FSImageCompression createCompression(Configuration conf) throws IOException { boolean compressImage = conf.getBoolean( DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESS_DEFAULT); if (!compressImage) { return createNoopCompression(); } String codecClassName = conf.get( DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT); return createCompression(conf, codecClassName); }
if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.FSIMAGE_COMPRESSION, imgVersion)) { compression = FSImageCompression.readCompressionHeader(conf, in); } else { compression = FSImageCompression.createNoopCompression(); in = compression.unwrapInputStream(fin);