.withChecksumType(HStore.getChecksumType(conf)) .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blocksize) .withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true) .build(); halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
HFileContext meta = new HFileContextBuilder() .withIncludesTags(withTag) .withDataBlockEncoding(encoding) .build(); HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration))
HFileContextBuilder hcBuilder = new HFileContextBuilder(); hcBuilder.withBlockSize(2 * 1024); hcBuilder.withDataBlockEncoding(encoding); HFileContext hFileContext = hcBuilder.build(); StoreFileWriter writer = new StoreFileWriter.Builder(
private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) { if (compression == null) { compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; } HFileContext hFileContext = new HFileContextBuilder() .withIncludesMvcc(includeMVCCReadpoint) .withIncludesTags(includesTag) .withCompression(compression) .withCompressTags(family.isCompressTags()) .withChecksumType(checksumType) .withBytesPerCheckSum(bytesPerChecksum) .withBlockSize(blocksize) .withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()) .withEncryptionContext(cryptoContext) .withCreateTime(EnvironmentEdgeManager.currentTime()) .build(); return hFileContext; }
.withDataBlockEncoding(DataBlockEncoding.NONE).build(); HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, hfilePath).withFileContext(meta)
.withBytesPerCheckSum(CKBYTES).withChecksumType(ChecksumType.NULL) .withBlockSize(DATA_BLOCK_SIZE) .withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()) .withIncludesTags(useTags).build(); StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
.withCompressTags(true).withDataBlockEncoding(DataBlockEncoding.PREFIX).build();
.withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES) .withDataBlockEncoding(dataBlockEncoderAlgo) .build();
.withDataBlockEncoding(encoding) .withCompression(Compression.Algorithm.NONE) .withIncludesMvcc(includesMemstoreTS)
.withEncryptionContext(cryptoContext) .withCompression(compression) .withDataBlockEncoding(encoding) .build();
Path makeNewFile(TagUsage tagUsage) throws IOException { Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); writer.append(toKV("e", tagUsage)); writer.append(toKV("g", tagUsage)); // block transition writer.append(toKV("i", tagUsage)); writer.append(toKV("k", tagUsage)); writer.close(); fout.close(); return ncTFile; }
.withDataBlockEncoding(fileContext.getDataBlockEncoding()) .withHBaseCheckSum(fileContext.isUseHBaseChecksum()) .withCompressTags(fileContext.isCompressTags())
.withBytesPerCheckSum(CKBYTES).withChecksumType(ChecksumType.NULL) .withBlockSize(DATA_BLOCK_SIZE) .withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()) .withIncludesTags(useTags).build(); final boolean cacheBlocks = false;
.withBytesPerCheckSum(bytesPerChecksum) .withBlockSize(blocksize) .withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()) .withEncryptionContext(cryptoContext) .withCreateTime(EnvironmentEdgeManager.currentTime()).build();
.withBlockSize(family.getBlocksize()) .withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()) .withEncryptionContext(Encryption.Context.NONE) .withCreateTime(EnvironmentEdgeManager.currentTime())
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)) .withBlockSize(blockSize); contextBuilder.withDataBlockEncoding(encoding); HFileContext hFileContext = contextBuilder.build();
private HFileContext getContext(Configuration conf) { HFileContextBuilder contextBuilder = new HFileContextBuilder(); String compressionType = conf.get(HCOLUMN_DESCRIPTOR_COMPRESSION_TYPE_KEY); if (compressionType != null) { contextBuilder.withCompression(HFileWriterImpl.compressionByName(compressionType)); } String dataBlockEncoding = conf.get(HCOLUMN_DESCRIPTOR_DATA_BLOCK_ENCODING_KEY); if (dataBlockEncoding != null) { contextBuilder.withDataBlockEncoding(DataBlockEncoding.valueOf(dataBlockEncoding)); } return contextBuilder.build(); } }
private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) { if (compression == null) { compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; } HFileContext hFileContext = new HFileContextBuilder() .withIncludesMvcc(includeMVCCReadpoint) .withIncludesTags(includesTag) .withCompression(compression) .withCompressTags(family.isCompressTags()) .withChecksumType(checksumType) .withBytesPerCheckSum(bytesPerChecksum) .withBlockSize(blocksize) .withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()) .withEncryptionContext(cryptoContext) .withCreateTime(EnvironmentEdgeManager.currentTime()) .build(); return hFileContext; }
Path makeNewFile(TagUsage tagUsage) throws IOException { Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); writer.append(toKV("e", tagUsage)); writer.append(toKV("g", tagUsage)); // block transition writer.append(toKV("i", tagUsage)); writer.append(toKV("k", tagUsage)); writer.close(); fout.close(); return ncTFile; }
public HFileSortedOplogWriter() throws IOException { HFileContext hcontext = new HFileContextBuilder() .withBlockSize(sopConfig.getBlockSize()) .withBytesPerCheckSum(sopConfig.getBytesPerChecksum()) .withChecksumType(HFileSortedOplogFactory.convertChecksum( sopConfig.getChecksum())) .withCompression(HFileSortedOplogFactory.convertCompression( sopConfig.getCompression())) .withDataBlockEncoding(HFileSortedOplogFactory.convertEncoding( sopConfig.getKeyEncoding()).getDataBlockEncoding()) .build(); writer = HFile.getWriterFactory(hconf, hcache) .withPath(fs, path) .withFileContext(hcontext) .withComparator(sopConfig.getComparator()) .create(); bfw = sopConfig.isBloomFilterEnabled() ? // BloomFilterFactory.createGeneralBloomAtWrite(hconf, hcache, BloomType.ROW, // 0, writer, sopConfig.getComparator()) BloomFilterFactory.createGeneralBloomAtWrite(hconf, hcache, BloomType.ROW, 0, writer) : null; }