Refine search
/** * This utility method creates a new Hbase HColumnDescriptor object based on a * Thrift ColumnDescriptor "struct". * * @param in Thrift ColumnDescriptor object * @return HColumnDescriptor * @throws IllegalArgument if the column name is empty */ static public HColumnDescriptor colDescFromThrift(ColumnDescriptor in) throws IllegalArgument { Compression.Algorithm comp = Compression.getCompressionAlgorithmByName(in.compression.toLowerCase(Locale.ROOT)); BloomType bt = BloomType.valueOf(in.bloomFilterType); if (in.name == null || !in.name.hasRemaining()) { throw new IllegalArgument("column name is empty"); } byte [] parsedName = CellUtil.parseColumn(Bytes.getBytes(in.name))[0]; HColumnDescriptor col = new HColumnDescriptor(parsedName) .setMaxVersions(in.maxVersions) .setCompressionType(comp) .setInMemory(in.inMemory) .setBlockCacheEnabled(in.blockCacheEnabled) .setTimeToLive(in.timeToLive > 0 ? in.timeToLive : Integer.MAX_VALUE) .setBloomFilterType(bt); return col; }
private void parseColumnFamilyOptions(CommandLine cmd) { String dataBlockEncodingStr = cmd.getOptionValue(HFileTestUtil.OPT_DATA_BLOCK_ENCODING); dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null : DataBlockEncoding.valueOf(dataBlockEncodingStr); String compressStr = cmd.getOptionValue(OPT_COMPRESSION); compressAlgo = compressStr == null ? Compression.Algorithm.NONE : Compression.Algorithm.valueOf(compressStr); String bloomStr = cmd.getOptionValue(OPT_BLOOM); bloomType = bloomStr == null ? BloomType.ROW : BloomType.valueOf(bloomStr); if (bloomType == BloomType.ROWPREFIX_FIXED_LENGTH) { if (!cmd.hasOption(OPT_BLOOM_PARAM)) { LOG.error("the parameter of bloom filter {} is not specified", bloomType.name()); } else { conf.set(BloomFilterUtil.PREFIX_LENGTH_KEY, cmd.getOptionValue(OPT_BLOOM_PARAM)); } } if (bloomType == BloomType.ROWPREFIX_DELIMITED) { if (!cmd.hasOption(OPT_BLOOM_PARAM)) { LOG.error("the parameter of bloom filter {} is not specified", bloomType.name()); } else { conf.set(BloomFilterUtil.DELIMITER_KEY, cmd.getOptionValue(OPT_BLOOM_PARAM)); } } inMemoryCF = cmd.hasOption(OPT_INMEMORY); if (cmd.hasOption(OPT_ENCRYPTION)) { cipher = Encryption.getCipher(conf, cmd.getOptionValue(OPT_ENCRYPTION)); } }
/** * This utility method creates a new Thrift ColumnDescriptor "struct" based on * an Hbase HColumnDescriptor object. * * @param in * Hbase HColumnDescriptor object * @return Thrift ColumnDescriptor */ static public ColumnDescriptor colDescFromHbase(HColumnDescriptor in) { ColumnDescriptor col = new ColumnDescriptor(); col.name = ByteBuffer.wrap(Bytes.add(in.getName(), KeyValue.COLUMN_FAMILY_DELIM_ARRAY)); col.maxVersions = in.getMaxVersions(); col.compression = in.getCompressionType().toString(); col.inMemory = in.isInMemory(); col.blockCacheEnabled = in.isBlockCacheEnabled(); col.bloomFilterType = in.getBloomFilterType().toString(); col.timeToLive = in.getTimeToLive(); return col; }
private static HColumnDescriptor getHColumnDesciptor(ColumnFamilyDescriptor descriptor) { HColumnDescriptor hFamily = new HColumnDescriptor(descriptor.getName()); hFamily.setMaxVersions(descriptor.getMaxVersions()); hFamily.setCompressionType(Compression.Algorithm.valueOf(descriptor.getCompressionType().name())); hFamily.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf( descriptor.getBloomType().name())); for (Map.Entry<String, String> property : descriptor.getProperties().entrySet()) { hFamily.setValue(property.getKey(), property.getValue()); } return hFamily; }
/** * Create a set of column descriptors with the combination of compression, * encoding, bloom codecs available. * @param prefix family names prefix * @return the list of column descriptors */ public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) { List<HColumnDescriptor> htds = new ArrayList<>(); long familyId = 0; for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) { for (DataBlockEncoding encodingType: DataBlockEncoding.values()) { for (BloomType bloomType: BloomType.values()) { String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId); HColumnDescriptor htd = new HColumnDescriptor(name); htd.setCompressionType(compressionType); htd.setDataBlockEncoding(encodingType); htd.setBloomFilterType(bloomType); htds.add(htd); familyId++; } } } return htds; }
String cfName = hcdt.getNameAsString(); String cfQualifiedName = getColumnFamilyQualifiedName(clusterName, nameSpace, tableName, cfName); ret.setAttribute(DESCRIPTION_ATTR, cfName); ret.setAttribute(OWNER, owner); ret.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, hcdt.isBlockCacheEnabled()); ret.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, (hcdt.getBloomFilterType() != null ? hcdt.getBloomFilterType().name():null)); ret.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, hcdt.isCacheBloomsOnWrite()); ret.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, hcdt.isCacheDataOnWrite());
private static ColumnFamilyDescriptor getColumnFamilyDescriptor(HColumnDescriptor descriptor) { String name = descriptor.getNameAsString(); int maxVersions = descriptor.getMaxVersions(); ColumnFamilyDescriptor.CompressionType compressionType = ColumnFamilyDescriptor.CompressionType.valueOf(descriptor.getCompressionType().getName().toUpperCase()); ColumnFamilyDescriptor.BloomType bloomType = ColumnFamilyDescriptor.BloomType.valueOf(descriptor.getBloomFilterType().name().toUpperCase()); Map<String, String> properties = new HashMap<>(); for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> value : descriptor.getValues().entrySet()) { properties.put(org.apache.hadoop.hbase.util.Bytes.toString(value.getKey().get()), org.apache.hadoop.hbase.util.Bytes.toString(value.getValue().get())); } return new ColumnFamilyDescriptor(name, maxVersions, compressionType, bloomType, properties); } }
final boolean blockCacheEnabled, final int blocksize, final int timeToLive, final String bloomFilter, final int scope) { isLegalFamilyName(familyName); this.name = familyName; setMaxVersions(maxVersions); setMinVersions(minVersions); setKeepDeletedCells(keepDeletedCells); setInMemory(inMemory); valueOf(compression.toUpperCase())); setDataBlockEncoding(DataBlockEncoding. valueOf(dataBlockEncoding.toUpperCase())); setBloomFilterType(BloomType. valueOf(bloomFilter.toUpperCase())); setBlocksize(blocksize); setScope(scope);
setMaxVersions(in.readInt()); int ordinal = in.readInt(); setCompressionType(Compression.Algorithm.values()[ordinal]); setInMemory(in.readBoolean()); setBloomFilterType(in.readBoolean() ? BloomType.ROW : BloomType.NONE); if (getBloomFilterType() != BloomType.NONE && version < 5) { value.set(Bytes.toBytes( Boolean.getBoolean(Bytes.toString(value.get())) ? BloomType.ROW.toString() : BloomType.NONE.toString()));
columnDescriptor.setCompressionType(Algorithm.valueOf(compression)); if(blockCache != null) columnDescriptor.setBlockCacheEnabled(Boolean.parseBoolean(blockCache)); if(blockSize != null) columnDescriptor.setBlocksize(Integer.parseInt(blockSize)); if(bloomFilter != null) columnDescriptor.setBloomFilterType(BloomType.valueOf(bloomFilter)); if(maxVersions != null) columnDescriptor.setMaxVersions(Integer.parseInt(maxVersions));
assertEquals("Incorrect bloom filter used for column family " + familyStr + "(reader: " + reader + ")", hcd.getBloomFilterType(), BloomType.valueOf(Bytes.toString(bloomFilter))); assertEquals("Incorrect compression used for column family " + familyStr + "(reader: " + reader + ")", hcd.getCompressionType(), reader.getFileContext().getCompression());
bloomTypeConfigValue.append('&'); bloomTypeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); bloomTypeConfigValue.append('='); String bloomType = familyDescriptor.getBloomFilterType().toString(); if (bloomType == null) { bloomType = HColumnDescriptor.DEFAULT_BLOOMFILTER;
@Override public BloomType getBloomFilterType() { return getStringOrDefault(BLOOMFILTER_BYTES, n -> BloomType.valueOf(n.toUpperCase()), DEFAULT_BLOOMFILTER); }
public HFileTarget(Path path, HColumnDescriptor hcol) { super(path, HFileOutputFormatForCrunch.class, SequentialFileNamingScheme.getInstance()); if (hcol != null) { outputConf(HFileOutputFormatForCrunch.HCOLUMN_DESCRIPTOR_COMPRESSION_TYPE_KEY, hcol.getCompressionType().getName()); outputConf(HFileOutputFormatForCrunch.HCOLUMN_DESCRIPTOR_DATA_BLOCK_ENCODING_KEY, hcol.getDataBlockEncoding().name()); outputConf(HFileOutputFormatForCrunch.HCOLUMN_DESCRIPTOR_BLOOM_FILTER_TYPE_KEY, hcol.getBloomFilterType().name()); } }
options.addOption(BLOOM_FILTER_OPTION, "bloom_filter", true, "Bloom filter type, one of " + Arrays.toString(BloomType.values())); options.addOption(BLOOM_FILTER_PARAM_OPTION, "bloom_param", true, "the parameter of the bloom filter"); bloomType = BloomType.valueOf(cmdLine.getOptionValue( BLOOM_FILTER_OPTION));
+ bloomFilterType.toString() + ":" + prefixLengthString; throw new IllegalArgumentException(message, nfe);
public ModifyableColumnFamilyDescriptor setBloomFilterType(final BloomType bt) { return setValue(BLOOMFILTER_BYTES, bt.name()); }
public static Collection<Object[]> generateParams(Compression.Algorithm algo, boolean useDataBlockEncoding) { List<Object[]> parameters = new ArrayList<>(); for (BloomType bloomType : BloomType.values()) { DataBlockEncoding dataBlockEncoding = useDataBlockEncoding ? DataBlockEncoding.PREFIX : DataBlockEncoding.NONE; parameters.add(new Object[] { algo, bloomType, dataBlockEncoding }); } return parameters; }