public Map<byte[], byte[]> loadFileInfo() throws IOException { Map<byte [], byte []> fi = reader.loadFileInfo(); byte[] b = fi.get(BLOOM_FILTER_TYPE_KEY); if (b != null) { bloomFilterType = BloomType.valueOf(Bytes.toString(b)); } lastBloomKey = fi.get(LAST_BLOOM_KEY); byte[] cnt = fi.get(DELETE_FAMILY_COUNT); if (cnt != null) { deleteFamilyCnt = Bytes.toLong(cnt); } return fi; }
/** * Create a set of column descriptors with the combination of compression, * encoding, bloom codecs available. * * @param prefix family names prefix * @return the list of column descriptors */ public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) { List<HColumnDescriptor> htds = new ArrayList<HColumnDescriptor>(); long familyId = 0; for (Compression.Algorithm compressionType : getSupportedCompressionAlgorithms()) { for (DataBlockEncoding encodingType : DataBlockEncoding.values()) { for (StoreFile.BloomType bloomType : StoreFile.BloomType.values()) { String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId); HColumnDescriptor htd = new HColumnDescriptor(name); htd.setCompressionType(compressionType); htd.setDataBlockEncoding(encodingType); htd.setBloomFilterType(bloomType); htds.add(htd); familyId++; } } } return htds; }
LOG.debug("Compacting " + file + ", keycount=" + keyCount + ", bloomtype=" + r.getBloomFilterType().toString() + ", size=" + StringUtils.humanReadableInt(r.length()) + ", encoding=" + r.getHFileReader().getEncodingOnDisk() +
generalBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader); LOG.info("Loaded " + bloomFilterType.toString() + " (" + generalBloomFilter.getClass().getSimpleName() + ") metadata for " + reader.getName());
columnDescriptor.setBlocksize(Integer.parseInt(blockSize)); if(bloomFilter != null) columnDescriptor.setBloomFilterType(BloomType.valueOf(bloomFilter)); if(maxVersions != null) columnDescriptor.setMaxVersions(Integer.parseInt(maxVersions));
String bloomType = familyDescriptor.getBloomFilterType().toString(); if (bloomType == null) { bloomType = HColumnDescriptor.DEFAULT_BLOOMFILTER;
/** * This utility method creates a new Hbase HColumnDescriptor object based on a * Thrift ColumnDescriptor "struct". * * @param in * Thrift ColumnDescriptor object * @return HColumnDescriptor * @throws IllegalArgument */ static public HColumnDescriptor colDescFromThrift(ColumnDescriptor in) throws IllegalArgument { Compression.Algorithm comp = Compression.getCompressionAlgorithmByName(in.compression.toLowerCase()); StoreFile.BloomType bt = BloomType.valueOf(in.bloomFilterType); if (in.name == null || !in.name.hasRemaining()) { throw new IllegalArgument("column name is empty"); } byte [] parsedName = KeyValue.parseColumn(Bytes.getBytes(in.name))[0]; HColumnDescriptor col = new HColumnDescriptor(parsedName) .setMaxVersions(in.maxVersions) .setCompressionType(comp) .setInMemory(in.inMemory) .setBlockCacheEnabled(in.blockCacheEnabled) .setTimeToLive(in.timeToLive) .setBloomFilterType(bt); return col; }
private WriterLength getNewWriter(byte[] family, Configuration conf) throws IOException { WriterLength wl = new WriterLength(); Path familydir = new Path(outputdir, Bytes.toString(family)); String compression = compressionMap.get(family); compression = compression == null ? defaultCompression : compression; String bloomTypeStr = bloomTypeMap.get(family); BloomType bloomType = BloomType.NONE; if (bloomTypeStr != null) { bloomType = BloomType.valueOf(bloomTypeStr); } Configuration tempConf = new Configuration(conf); tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blocksize) .withOutputDir(familydir) .withCompression(AbstractHFileWriter.compressionByName(compression)) .withBloomType(bloomType) .withComparator(KeyValue.COMPARATOR) .withDataBlockEncoder(encoder) .withChecksumType(Store.getChecksumType(conf)) .withBytesPerChecksum(Store.getBytesPerChecksum(conf)) .build(); this.writers.put(family, wl); return wl; }
/** * This utility method creates a new Thrift ColumnDescriptor "struct" based on * an Hbase HColumnDescriptor object. * * @param in * Hbase HColumnDescriptor object * @return Thrift ColumnDescriptor */ static public ColumnDescriptor colDescFromHbase(HColumnDescriptor in) { ColumnDescriptor col = new ColumnDescriptor(); col.name = ByteBuffer.wrap(Bytes.add(in.getName(), KeyValue.COLUMN_FAMILY_DELIM_ARRAY)); col.maxVersions = in.getMaxVersions(); col.compression = in.getCompression().toString(); col.inMemory = in.isInMemory(); col.blockCacheEnabled = in.isBlockCacheEnabled(); col.bloomFilterType = in.getBloomFilterType().toString(); return col; }
public static ColumnFamilyConfig buildCfConfig(Conf conf) { ColumnFamilyConfig family = new ColumnFamilyConfig(); String compression = conf.getChild("compression").getValue(null); if (compression != null) { family.setCompression(Compression.Algorithm.valueOf(compression.toUpperCase())); } String bloomFilter = conf.getChild("bloomFilter").getValue(null); if (bloomFilter != null) { family.setBoomFilter(StoreFile.BloomType.valueOf(bloomFilter.toUpperCase())); } Integer blockSize = conf.getChild("blockSize").getValueAsInteger(null); if (blockSize != null) { family.setBlockSize(blockSize); } return family; } }
private boolean closeGeneralBloomFilter() throws IOException { boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter); // add the general Bloom filter writer and append file info if (hasGeneralBloom) { writer.addGeneralBloomFilter(generalBloomFilterWriter); writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY, Bytes.toBytes(bloomType.toString())); if (lastBloomKey != null) { writer.appendFileInfo(LAST_BLOOM_KEY, Arrays.copyOfRange( lastBloomKey, lastBloomKeyOffset, lastBloomKeyOffset + lastBloomKeyLen)); } } return hasGeneralBloom; }
/** * Create all combinations of Bloom filters and compression algorithms for * testing. */ private static List<Object[]> bloomAndCompressionCombinations() { List<Object[]> configurations = new ArrayList<Object[]>(); for (Compression.Algorithm comprAlgo : HBaseTestingUtility.COMPRESSION_ALGORITHMS) { for (StoreFile.BloomType bloomType : StoreFile.BloomType.values()) { configurations.add(new Object[]{comprAlgo, bloomType}); } } return Collections.unmodifiableList(configurations); }
/** * @param bt bloom filter type * @return this (for chained invocation) */ public HColumnDescriptor setBloomFilterType(final StoreFile.BloomType bt) { return setValue(BLOOMFILTER, bt.toString()); }
/** * @return bloom filter type used for new StoreFiles in ColumnFamily */ public StoreFile.BloomType getBloomFilterType() { String n = getValue(BLOOMFILTER); if (n == null) { n = DEFAULT_BLOOMFILTER; } return StoreFile.BloomType.valueOf(n.toUpperCase()); }