Refine search
@Override public void setCompression(HColumnDescriptor cd, String algo) { cd.setCompressionType(Compression.Algorithm.valueOf(algo)); }
compressionConfigValue.append('&'); compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); compressionConfigValue.append('='); compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getCompression().getName(), "UTF-8"));
public byte[] createTestV1Block(Compression.Algorithm algo) throws IOException { Compressor compressor = algo.getCompressor(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream os = algo.createCompressionStream(baos, compressor, 0); DataOutputStream dos = new DataOutputStream(os); BlockType.META.write(dos); // Let's make this a meta block. writeTestBlockContents(dos); dos.flush(); algo.returnCompressor(compressor); return baos.toByteArray(); }
private static HColumnDescriptor getHColumnDesciptor(ColumnFamilyDescriptor descriptor) { HColumnDescriptor hFamily = new HColumnDescriptor(descriptor.getName()); hFamily.setMaxVersions(descriptor.getMaxVersions()); hFamily.setCompressionType(Compression.Algorithm.valueOf(descriptor.getCompressionType().name())); hFamily.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf( descriptor.getBloomType().name())); for (Map.Entry<String, String> property : descriptor.getProperties().entrySet()) { hFamily.setValue(property.getKey(), property.getValue()); } return hFamily; }
private static void createTable(HBaseGraphConfiguration config, Admin admin, String name, int ttl) throws IOException { TableName tableName = getTableName(config, name); if (admin.tableExists(tableName)) return; try { HTableDescriptor tableDescriptor = new HTableDescriptor(tableName); tableDescriptor.setDurability(config.getTableAsyncWAL() ? Durability.ASYNC_WAL : Durability.USE_DEFAULT); HColumnDescriptor columnDescriptor = new HColumnDescriptor(DEFAULT_FAMILY) .setCompressionType(Compression.Algorithm.valueOf(config.getCompressionAlgorithm().toUpperCase())) .setBloomFilterType(BloomType.ROW) .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF) .setMaxVersions(1) .setMinVersions(0) .setBlocksize(32768) .setBlockCacheEnabled(true) .setTimeToLive(ttl); tableDescriptor.addFamily(columnDescriptor); int regionCount = config.getRegionCount(); if (regionCount <= 1) { admin.createTable(tableDescriptor); } else { admin.createTable(tableDescriptor, getStartKey(regionCount), getEndKey(regionCount), regionCount); } } catch (IOException e) { LOGGER.error("Could not create table " + tableName, e); throw e; } }
String cfName = hcdt.getNameAsString(); String cfQualifiedName = getColumnFamilyQualifiedName(clusterName, nameSpace, tableName, cfName); ret.setAttribute(DESCRIPTION_ATTR, cfName); ret.setAttribute(OWNER, owner); ret.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, hcdt.isBlockCacheEnabled()); ret.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, (hcdt.getBloomFilterType() != null ? hcdt.getBloomFilterType().name():null)); ret.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, hcdt.isCacheBloomsOnWrite()); ret.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, hcdt.isCacheDataOnWrite()); ret.setAttribute(ATTR_CF_CACHED_INDEXES_ON_WRITE, hcdt.isCacheIndexesOnWrite()); ret.setAttribute(ATTR_CF_COMPACTION_COMPRESSION_TYPE, (hcdt.getCompactionCompressionType() != null ? hcdt.getCompactionCompressionType().name():null)); ret.setAttribute(ATTR_CF_COMPRESSION_TYPE, (hcdt.getCompressionType() != null ? hcdt.getCompressionType().name():null)); ret.setAttribute(ATTR_CF_DATA_BLOCK_ENCODING, (hcdt.getDataBlockEncoding() != null ? hcdt.getDataBlockEncoding().name():null)); ret.setAttribute(ATTR_CF_ENCRYPTION_TYPE, hcdt.getEncryptionType());
private static ColumnFamilyDescriptor getColumnFamilyDescriptor(HColumnDescriptor descriptor) { String name = descriptor.getNameAsString(); int maxVersions = descriptor.getMaxVersions(); ColumnFamilyDescriptor.CompressionType compressionType = ColumnFamilyDescriptor.CompressionType.valueOf(descriptor.getCompressionType().getName().toUpperCase()); ColumnFamilyDescriptor.BloomType bloomType = ColumnFamilyDescriptor.BloomType.valueOf(descriptor.getBloomFilterType().name().toUpperCase()); Map<String, String> properties = new HashMap<>(); for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> value : descriptor.getValues().entrySet()) { properties.put(org.apache.hadoop.hbase.util.Bytes.toString(value.getKey().get()), org.apache.hadoop.hbase.util.Bytes.toString(value.getValue().get())); } return new ColumnFamilyDescriptor(name, maxVersions, compressionType, bloomType, properties); } }
columnDescriptor.setCompressionType(Algorithm.valueOf(compression)); if(blockCache != null) columnDescriptor.setBlockCacheEnabled(Boolean.parseBoolean(blockCache)); if(blockSize != null) columnDescriptor.setBlocksize(Integer.parseInt(blockSize)); if(bloomFilter != null) columnDescriptor.setBloomFilterType(BloomType.valueOf(bloomFilter));
for (HColumnDescriptor columnDescriptor : hTableDescriptor.getColumnFamilies()) if (columnDescriptor.getNameAsString().equalsIgnoreCase("HBASEUSERXYZ")) Assert.assertEquals(Algorithm.valueOf("GZ"), columnDescriptor.getCompactionCompressionType()); Assert.assertEquals(Integer.parseInt("12345678"), columnDescriptor.getTimeToLive()); Assert.assertEquals(Algorithm.valueOf("GZ"), columnDescriptor.getCompressionType()); Assert.assertEquals(Integer.parseInt("6"), columnDescriptor.getMaxVersions()); Assert.assertEquals(Integer.parseInt("3"), columnDescriptor.getMinVersions());
@Override public void bulkHFile(final HColumnDescriptor hcd, final FileStatus hfileStatus) throws IOException { Path hfile = hfileStatus.getPath(); HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(getConf()), getConf()); try { if (hcd.getCompressionType() != reader.getFileContext().getCompression()) { hcd.setCompressionType(reader.getFileContext().getCompression()); LOG.info("Setting compression " + hcd.getCompressionType().name() + " for family " + hcd.toString()); } reader.loadFileInfo(); byte[] first = reader.getFirstRowKey(); byte[] last = reader.getLastRowKey(); LOG.info("Trying to figure out region boundaries hfile=" + hfile + " first=" + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last)); // To eventually infer start key-end key boundaries Integer value = map.containsKey(first)? map.get(first):0; map.put(first, value+1); value = map.containsKey(last)? map.get(last):0; map.put(last, value-1); } finally { reader.close(); } } });
public HFileTarget(Path path, HColumnDescriptor hcol) { super(path, HFileOutputFormatForCrunch.class, SequentialFileNamingScheme.getInstance()); if (hcol != null) { outputConf(HFileOutputFormatForCrunch.HCOLUMN_DESCRIPTOR_COMPRESSION_TYPE_KEY, hcol.getCompressionType().getName()); outputConf(HFileOutputFormatForCrunch.HCOLUMN_DESCRIPTOR_DATA_BLOCK_ENCODING_KEY, hcol.getDataBlockEncoding().name()); outputConf(HFileOutputFormatForCrunch.HCOLUMN_DESCRIPTOR_BLOOM_FILTER_TYPE_KEY, hcol.getBloomFilterType().name()); } }
/** * Compression types supported in hbase. * LZO is not bundled as part of the hbase distribution. * See <a href="http://wiki.apache.org/hadoop/UsingLzoCompression">LZO Compression</a> * for how to enable it. * @param type Compression type setting. * @return this (for chained invocation) */ public HColumnDescriptor setCompressionType(Compression.Algorithm type) { return setValue(COMPRESSION, type.getName().toUpperCase()); }
/** @return compression type being used for the column family for major compression */ public Compression.Algorithm getCompactionCompression() { String n = getValue(COMPRESSION_COMPACT); if (n == null) { return getCompression(); } return Compression.Algorithm.valueOf(n.toUpperCase()); }
@Override public String toString() { return "writer=" + (path != null ? path.toString() : null) + ", name=" + name + ", compression=" + hFileContext.getCompression().getName(); }
@Override public Compression.Algorithm getCompressionType() { return getStringOrDefault(COMPRESSION_BYTES, n -> Compression.Algorithm.valueOf(n.toUpperCase()), DEFAULT_COMPRESSION); }
/** * @param compressionAlgorithmName What kind of algorithm should be used * as baseline for comparison (e.g. lzo, gz). */ public DataBlockEncodingTool(String compressionAlgorithmName) { this.compressionAlgorithmName = compressionAlgorithmName; this.compressionAlgorithm = Compression.getCompressionAlgorithmByName( compressionAlgorithmName); this.compressor = this.compressionAlgorithm.getCompressor(); this.decompressor = this.compressionAlgorithm.getDecompressor(); }
@Override public Compression.Algorithm getCompactionCompressionType() { return getStringOrDefault(COMPRESSION_COMPACT_BYTES, n -> Compression.Algorithm.valueOf(n.toUpperCase()), getCompressionType()); }
private static void testDataBlockingTool(Path path) throws IOException { Configuration conf = HBaseConfiguration.create(); int maxKV = Integer.MAX_VALUE; boolean doVerify = true; boolean doBenchmark = true; String testHFilePath = path.toString(); DataBlockEncodingTool.testCodecs(conf, maxKV, testHFilePath, Compression.Algorithm.GZ.getName(), doBenchmark, doVerify); }
/** * Releases the compressor this writer uses to compress blocks into the * compressor pool. */ @Override public void close() { if (compressor != null) { this.fileContext.getCompression().returnCompressor(compressor); compressor = null; } }