/** * Serialize column family to block size map to configuration. * Invoked while configuring the MR job for incremental load. * @param tableDescriptor to read the properties from * @param conf to persist serialized values into * * @throws IOException * on failure to read column family descriptors */ @VisibleForTesting static void configureBlockSize(HTableDescriptor tableDescriptor, Configuration conf) throws UnsupportedEncodingException { StringBuilder blockSizeConfigValue = new StringBuilder(); if (tableDescriptor == null) { // could happen with mock table instance return; } Collection<HColumnDescriptor> families = tableDescriptor.getFamilies(); int i = 0; for (HColumnDescriptor familyDescriptor : families) { if (i++ > 0) { blockSizeConfigValue.append('&'); } blockSizeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); blockSizeConfigValue.append('='); blockSizeConfigValue.append(URLEncoder.encode(String.valueOf(familyDescriptor.getBlocksize()), "UTF-8")); } // Get rid of the last ampersand conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString()); }
@Test(expected=IllegalArgumentException.class) public void testAddDuplicateFamilies() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); htd.addFamily(hcd); assertEquals(1000, htd.getFamily(familyName).getBlocksize()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); htd.addFamily(hcd); }
@Test public void testModifyFamily() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); hcd.setDFSReplication((short) 3); htd.addFamily(hcd); assertEquals(1000, htd.getFamily(familyName).getBlocksize()); assertEquals(3, htd.getFamily(familyName).getDFSReplication()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); hcd.setDFSReplication((short) 1); htd.modifyFamily(hcd); assertEquals(2000, htd.getFamily(familyName).getBlocksize()); assertEquals(1, htd.getFamily(familyName).getDFSReplication()); }
@Test public void testModifyColumnFamily() throws IOException { Admin admin = TEST_UTIL.getAdmin(); HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_0); int blockSize = cfDescriptor.getBlocksize(); // Create a table with one families HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); baseHtd.addFamily(cfDescriptor); admin.createTable(baseHtd); admin.disableTable(TABLE_NAME); try { // Verify the table descriptor verifyTableDescriptor(TABLE_NAME, FAMILY_0); int newBlockSize = 2 * blockSize; cfDescriptor.setBlocksize(newBlockSize); // Modify colymn family admin.modifyColumnFamily(TABLE_NAME, cfDescriptor); HTableDescriptor htd = admin.getTableDescriptor(TABLE_NAME); HColumnDescriptor hcfd = htd.getFamily(FAMILY_0); assertTrue(hcfd.getBlocksize() == newBlockSize); } finally { admin.deleteTable(TABLE_NAME); } }
@Test public void testModifyNonExistingColumnFamily() throws IOException { Admin admin = TEST_UTIL.getAdmin(); HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_1); int blockSize = cfDescriptor.getBlocksize(); // Create a table with one families HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); baseHtd.addFamily(new HColumnDescriptor(FAMILY_0)); admin.createTable(baseHtd); admin.disableTable(TABLE_NAME); try { // Verify the table descriptor verifyTableDescriptor(TABLE_NAME, FAMILY_0); int newBlockSize = 2 * blockSize; cfDescriptor.setBlocksize(newBlockSize); // Modify a column family that is not in the table. try { admin.modifyColumnFamily(TABLE_NAME, cfDescriptor); Assert.fail("Modify a non-exist column family should fail"); } catch (InvalidFamilyOperationException e) { // Expected. } } finally { admin.deleteTable(TABLE_NAME); } }
@Test(expected=IllegalArgumentException.class) public void testAddDuplicateFamilies() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); htd.addFamily(hcd); assertEquals(1000, htd.getFamily(familyName).getBlocksize()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); htd.addFamily(hcd); }
HColumnDescriptor deserializedHcd = HColumnDescriptor.parseFrom(bytes); assertTrue(hcd.equals(deserializedHcd)); assertEquals(v, hcd.getBlocksize()); assertEquals(v, hcd.getTimeToLive()); assertEquals(v, hcd.getScope());
@Test public void testModifyFamily() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); hcd.setDFSReplication((short) 3); htd.addFamily(hcd); assertEquals(1000, htd.getFamily(familyName).getBlocksize()); assertEquals(3, htd.getFamily(familyName).getDFSReplication()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); hcd.setDFSReplication((short) 1); htd.modifyFamily(hcd); assertEquals(2000, htd.getFamily(familyName).getBlocksize()); assertEquals(1, htd.getFamily(familyName).getDFSReplication()); }
/** * Serialize column family to block size map to configuration. * Invoked while configuring the MR job for incremental load. * @param tableDescriptor to read the properties from * @param conf to persist serialized values into * * @throws IOException * on failure to read column family descriptors */ @VisibleForTesting static void configureBlockSize(HTableDescriptor tableDescriptor, Configuration conf) throws UnsupportedEncodingException { StringBuilder blockSizeConfigValue = new StringBuilder(); if (tableDescriptor == null) { // could happen with mock table instance return; } Collection<HColumnDescriptor> families = tableDescriptor.getFamilies(); int i = 0; for (HColumnDescriptor familyDescriptor : families) { if (i++ > 0) { blockSizeConfigValue.append('&'); } blockSizeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); blockSizeConfigValue.append('='); blockSizeConfigValue.append(URLEncoder.encode(String.valueOf(familyDescriptor.getBlocksize()), "UTF-8")); } // Get rid of the last ampersand conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString()); }
blockSizeConfigValue.append('='); blockSizeConfigValue.append(URLEncoder.encode( String.valueOf(familyDescriptor.getBlocksize()), "UTF-8"));
@Override public int compare(HColumnDescriptor o1, HColumnDescriptor o2) { int nameResult = Bytes.compareTo(o1.getName(), o2.getName()); if (nameResult != 0) { return nameResult; } int maxVersionsResult = Integer.valueOf(o1.getMaxVersions()).compareTo(o2.getMaxVersions()); if (maxVersionsResult != 0) { return maxVersionsResult; } int timeToLiveResult = Integer.valueOf(o1.getTimeToLive()).compareTo(o2.getTimeToLive()); if (timeToLiveResult != 0) { return timeToLiveResult; } int inMemoryResult = Boolean.valueOf(o1.isInMemory()).compareTo(o2.isInMemory()); if (inMemoryResult != 0) { return inMemoryResult; } int blockSizeResult = Integer.valueOf(o1.getBlocksize()).compareTo(o2.getBlocksize()); if (blockSizeResult != 0) { return blockSizeResult; } int bloomTypeResult = SchemaPlatformBridge.get().compareBloom(o1, o2); if (bloomTypeResult != 0) { return bloomTypeResult; } return SchemaPlatformBridge.get().compareCompression(o1, o2); } }
static public AFamilyDescriptor hcdToAFD(HColumnDescriptor hcd) throws IOException { AFamilyDescriptor afamily = new AFamilyDescriptor(); afamily.name = ByteBuffer.wrap(hcd.getName()); String compressionAlgorithm = hcd.getCompressionType().getName(); if (compressionAlgorithm == "LZO") { afamily.compression = ACompressionAlgorithm.LZO; } else if (compressionAlgorithm == "GZ") { afamily.compression = ACompressionAlgorithm.GZ; } else { afamily.compression = ACompressionAlgorithm.NONE; } afamily.maxVersions = hcd.getMaxVersions(); afamily.blocksize = hcd.getBlocksize(); afamily.inMemory = hcd.isInMemory(); afamily.timeToLive = hcd.getTimeToLive(); afamily.blockCacheEnabled = hcd.isBlockCacheEnabled(); return afamily; }
this.family = family; this.conf = conf; this.blocksize = family.getBlocksize();
HColumnDescriptor deserializedHcd = HColumnDescriptor.parseFrom(bytes); assertTrue(hcd.equals(deserializedHcd)); assertEquals(v, hcd.getBlocksize()); assertEquals(v, hcd.getTimeToLive()); assertEquals(v, hcd.getScope());
@Test public void testModifyColumnFamily() throws IOException { Admin admin = TEST_UTIL.getAdmin(); HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_0); int blockSize = cfDescriptor.getBlocksize(); // Create a table with one families HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); baseHtd.addFamily(cfDescriptor); admin.createTable(baseHtd); admin.disableTable(TABLE_NAME); try { // Verify the table descriptor verifyTableDescriptor(TABLE_NAME, FAMILY_0); int newBlockSize = 2 * blockSize; cfDescriptor.setBlocksize(newBlockSize); // Modify colymn family admin.modifyColumnFamily(TABLE_NAME, cfDescriptor); HTableDescriptor htd = admin.getTableDescriptor(TABLE_NAME); HColumnDescriptor hcfd = htd.getFamily(FAMILY_0); assertTrue(hcfd.getBlocksize() == newBlockSize); } finally { admin.deleteTable(TABLE_NAME); } }
@Test public void testModifyNonExistingColumnFamily() throws IOException { Admin admin = TEST_UTIL.getAdmin(); HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_1); int blockSize = cfDescriptor.getBlocksize(); // Create a table with one families HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); baseHtd.addFamily(new HColumnDescriptor(FAMILY_0)); admin.createTable(baseHtd); admin.disableTable(TABLE_NAME); try { // Verify the table descriptor verifyTableDescriptor(TABLE_NAME, FAMILY_0); int newBlockSize = 2 * blockSize; cfDescriptor.setBlocksize(newBlockSize); // Modify a column family that is not in the table. try { admin.modifyColumnFamily(TABLE_NAME, cfDescriptor); Assert.fail("Modify a non-exist column family should fail"); } catch (InvalidFamilyOperationException e) { // Expected. } } finally { admin.deleteTable(TABLE_NAME); } }
if (hcd.getBlocksize() < 1024 || hcd.getBlocksize() > 16 * 1024 * 1024) { String message = "Block size for column family " + hcd.getNameAsString() + " must be between 1K and 16MB.";
.addStringMap(family.getConfiguration()) .addWritableMap(family.getValues()); this.blocksize = family.getBlocksize();
Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo(); int blocksize = familyDescriptor.getBlocksize(); Algorithm compression = familyDescriptor.getCompression(); BloomType bloomFilterType = familyDescriptor.getBloomFilterType();
Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo(); int blocksize = familyDescriptor.getBlocksize(); Algorithm compression = familyDescriptor.getCompression(); BloomType bloomFilterType = familyDescriptor.getBloomFilterType();