@Override public String toString() { return "writer=" + (path != null ? path.toString() : null) + ", name=" + name + ", compression=" + hFileContext.getCompression().getName(); }
/** * Releases the compressor this writer uses to compress blocks into the * compressor pool. */ @Override public void close() { if (compressor != null) { this.fileContext.getCompression().returnCompressor(compressor); compressor = null; } }
/** * @return true when on-disk blocks from this file are compressed, and/or encrypted; * false otherwise. */ public boolean isCompressedOrEncrypted() { Compression.Algorithm compressAlgo = getCompression(); boolean compressed = compressAlgo != null && compressAlgo != Compression.Algorithm.NONE; Encryption.Context cryptoContext = getEncryptionContext(); boolean encrypted = cryptoContext != null && cryptoContext != Encryption.Context.NONE; return compressed || encrypted; }
@Override public void bulkHFile(ColumnFamilyDescriptorBuilder builder, FileStatus hfileStatus) throws IOException { Path hfile = hfileStatus.getPath(); try (HFile.Reader reader = HFile.createReader(fs, hfile, CacheConfig.DISABLED, true, getConf())) { if (builder.getCompressionType() != reader.getFileContext().getCompression()) { builder.setCompressionType(reader.getFileContext().getCompression()); LOG.info("Setting compression " + reader.getFileContext().getCompression().name() + " for family " + builder.getNameAsString()); } reader.loadFileInfo(); byte[] first = reader.getFirstRowKey().get(); byte[] last = reader.getLastRowKey().get(); LOG.info("Trying to figure out region boundaries hfile=" + hfile + " first=" + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last)); // To eventually infer start key-end key boundaries Integer value = map.containsKey(first) ? map.get(first) : 0; map.put(first, value + 1); value = map.containsKey(last) ? map.get(last) : 0; map.put(last, value - 1); } } });
this.fileContext = fileContext; Compression.Algorithm compressionAlgorithm = fileContext.getCompression() == null ? NONE : fileContext.getCompression(); if (compressionAlgorithm != NONE) { compressor = compressionAlgorithm.getCompressor();
protected void finishClose(FixedFileTrailer trailer) throws IOException { // Write out encryption metadata before finalizing if we have a valid crypto context Encryption.Context cryptoContext = hFileContext.getEncryptionContext(); if (cryptoContext != Encryption.Context.NONE) { // Wrap the context's key and write it as the encryption metadata, the wrapper includes // all information needed for decryption trailer.setEncryptionKey(EncryptionUtil.wrapKey(cryptoContext.getConf(), cryptoContext.getConf().get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cryptoContext.getKey())); } // Now we can finish the close trailer.setMetaIndexCount(metaNames.size()); trailer.setTotalUncompressedBytes(totalUncompressedBytes+ trailer.getTrailerSize()); trailer.setEntryCount(entryCount); trailer.setCompressionCodec(hFileContext.getCompression()); long startTime = System.currentTimeMillis(); trailer.serialize(outputStream); HFile.updateWriteLatency(System.currentTimeMillis() - startTime); if (closeOutputStream) { outputStream.close(); outputStream = null; } } }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
Compression.Algorithm compression = fileContext.getCompression(); assert blockBufferWithoutHeader.hasArray(); if (compression != Compression.Algorithm.NONE) {
hcd.getBloomFilterType(), BloomType.valueOf(Bytes.toString(bloomFilter))); assertEquals("Incorrect compression used for column family " + familyStr + "(reader: " + reader + ")", hcd.getCompressionType(), reader.getFileContext().getCompression());
int plaintextLength; if (fileContext.getCompression() != Compression.Algorithm.NONE) { compressedByteStream.reset(); compressionStream.resetState(); if (this.fileContext.getCompression() != NONE) { compressedByteStream.reset(); compressedByteStream.write(headerBytes);
.withBytesPerCheckSum(0) .withCompression(fileContext.getCompression()) .withDataBlockEncoding(fileContext.getDataBlockEncoding()) .withHBaseCheckSum(fileContext.isUseHBaseChecksum())
/** * Releases the compressor this writer uses to compress blocks into the * compressor pool. */ @Override public void close() { if (compressor != null) { this.fileContext.getCompression().returnCompressor(compressor); compressor = null; } }
/** * Releases the compressor this writer uses to compress blocks into the * compressor pool. */ @Override public void close() { if (compressor != null) { this.fileContext.getCompression().returnCompressor(compressor); compressor = null; } }
@Override public String toString() { return "writer=" + (path != null ? path.toString() : null) + ", name=" + name + ", compression=" + hFileContext.getCompression().getName(); }
/** * Releases the compressor this writer uses to compress blocks into the * compressor pool. */ @Override public void close() { if (compressor != null) { this.fileContext.getCompression().returnCompressor(compressor); compressor = null; } }
/** * @return true when on-disk blocks from this file are compressed, and/or encrypted; * false otherwise. */ public boolean isCompressedOrEncrypted() { Compression.Algorithm compressAlgo = getCompression(); boolean compressed = compressAlgo != null && compressAlgo != Compression.Algorithm.NONE; Encryption.Context cryptoContext = getEncryptionContext(); boolean encrypted = cryptoContext != null && cryptoContext != Encryption.Context.NONE; return compressed || encrypted; }
/** * @return true when on-disk blocks from this file are compressed, and/or encrypted; * false otherwise. */ public boolean isCompressedOrEncrypted() { Compression.Algorithm compressAlgo = getCompression(); boolean compressed = compressAlgo != null && compressAlgo != Compression.Algorithm.NONE; Encryption.Context cryptoContext = getEncryptionContext(); boolean encrypted = cryptoContext != null && cryptoContext != Encryption.Context.NONE; return compressed || encrypted; }
/** * @return true when on-disk blocks from this file are compressed, and/or encrypted; * false otherwise. */ public boolean isCompressedOrEncrypted() { Compression.Algorithm compressAlgo = getCompression(); boolean compressed = compressAlgo != null && compressAlgo != Compression.Algorithm.NONE; Encryption.Context cryptoContext = getEncryptionContext(); boolean encrypted = cryptoContext != null && cryptoContext != Encryption.Context.NONE; return compressed || encrypted; }
/** * Sets remaining trailer fields, writes the trailer to disk, and optionally * closes the output stream. */ protected void finishClose(FixedFileTrailer trailer) throws IOException { trailer.setMetaIndexCount(metaNames.size()); trailer.setTotalUncompressedBytes(totalUncompressedBytes+ trailer.getTrailerSize()); trailer.setEntryCount(entryCount); trailer.setCompressionCodec(hFileContext.getCompression()); trailer.serialize(outputStream); if (closeOutputStream) { outputStream.close(); outputStream = null; } }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }