/** * Serialize column family to block size map to configuration. * Invoked while configuring the MR job for incremental load. * @param tableDescriptor to read the properties from * @param conf to persist serialized values into * * @throws IOException * on failure to read column family descriptors */ @VisibleForTesting static void configureBlockSize(HTableDescriptor tableDescriptor, Configuration conf) throws UnsupportedEncodingException { StringBuilder blockSizeConfigValue = new StringBuilder(); if (tableDescriptor == null) { // could happen with mock table instance return; } Collection<HColumnDescriptor> families = tableDescriptor.getFamilies(); int i = 0; for (HColumnDescriptor familyDescriptor : families) { if (i++ > 0) { blockSizeConfigValue.append('&'); } blockSizeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); blockSizeConfigValue.append('='); blockSizeConfigValue.append(URLEncoder.encode(String.valueOf(familyDescriptor.getBlocksize()), "UTF-8")); } // Get rid of the last ampersand conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString()); }
compressionConfigValue.append('&'); compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); compressionConfigValue.append('='); compressionConfigValue.append(URLEncoder.encode(familyDescriptor.getCompression().getName(), "UTF-8"));
dataBlockEncodingConfigValue.append('&'); dataBlockEncodingConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); dataBlockEncodingConfigValue.append('='); DataBlockEncoding encoding = familyDescriptor.getDataBlockEncoding();
bloomTypeConfigValue.append('&'); bloomTypeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); bloomTypeConfigValue.append('='); String bloomType = familyDescriptor.getBloomFilterType().toString();
/** * Constructor * @param htd the table descriptor */ public TableSchemaModel(HTableDescriptor htd) { setName(htd.getTableName().getNameAsString()); for (Map.Entry<Bytes, Bytes> e: htd.getValues().entrySet()) { addAttribute(Bytes.toString(e.getKey().get()), Bytes.toString(e.getValue().get())); } for (HColumnDescriptor hcd: htd.getFamilies()) { ColumnSchemaModel columnModel = new ColumnSchemaModel(); columnModel.setName(hcd.getNameAsString()); for (Map.Entry<Bytes, Bytes> e: hcd.getValues().entrySet()) { columnModel.addAttribute(Bytes.toString(e.getKey().get()), Bytes.toString(e.getValue().get())); } addColumnFamily(columnModel); } }
private int countMobFiles() throws IOException { Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); if (fs.exists(mobDirPath)) { FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath); return files.length; } return 0; }
@Test public void testMobFamilyDelete() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = createTableDescriptor(tableName, true); HColumnDescriptor hcd = htd.getFamily(FAMILY); htd.addFamily(new HColumnDescriptor(Bytes.toBytes("family2"))); Table table = createTableWithOneFile(htd); try { // the mob file exists Assert.assertEquals(1, countMobFiles(tableName, hcd.getNameAsString())); Assert.assertEquals(0, countArchiveMobFiles(tableName, hcd.getNameAsString())); String fileName = assertHasOneMobRow(table, tableName, hcd.getNameAsString()); Assert.assertFalse(mobArchiveExist(tableName, hcd.getNameAsString(), fileName)); Assert.assertTrue(mobTableDirExist(tableName)); TEST_UTIL.getAdmin().deleteColumnFamily(tableName, FAMILY); Assert.assertEquals(0, countMobFiles(tableName, hcd.getNameAsString())); Assert.assertEquals(1, countArchiveMobFiles(tableName, hcd.getNameAsString())); Assert.assertTrue(mobArchiveExist(tableName, hcd.getNameAsString(), fileName)); Assert.assertFalse(mobColumnFamilyDirExist(tableName, hcd.getNameAsString())); } finally { table.close(); TEST_UTIL.deleteTable(tableName); } }
hcd.getNameAsString()); assertTrue(fs.exists(new Path(familyPath, fileName)));
private long countMobCellsInMetadata() throws IOException { long mobCellsCount = 0; Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); if (fs.exists(mobDirPath)) { FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath); for (FileStatus file : files) { HStoreFile sf = new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true); sf.initReader(); Map<byte[], byte[]> fileInfo = sf.getReader().loadFileInfo(); byte[] count = fileInfo.get(MOB_CELLS_COUNT); assertTrue(count != null); mobCellsCount += Bytes.toLong(count); } } return mobCellsCount; }
@Test public void testDeleteMobTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = createTableDescriptor(tableName, true); HColumnDescriptor hcd = htd.getFamily(FAMILY); String fileName = null; Table table = createTableWithOneFile(htd); try { // the mob file exists Assert.assertEquals(1, countMobFiles(tableName, hcd.getNameAsString())); Assert.assertEquals(0, countArchiveMobFiles(tableName, hcd.getNameAsString())); fileName = assertHasOneMobRow(table, tableName, hcd.getNameAsString()); Assert.assertFalse(mobArchiveExist(tableName, hcd.getNameAsString(), fileName)); Assert.assertTrue(mobTableDirExist(tableName)); } finally { table.close(); TEST_UTIL.deleteTable(tableName); } Assert.assertFalse(TEST_UTIL.getAdmin().tableExists(tableName)); Assert.assertEquals(0, countMobFiles(tableName, hcd.getNameAsString())); Assert.assertEquals(1, countArchiveMobFiles(tableName, hcd.getNameAsString())); Assert.assertTrue(mobArchiveExist(tableName, hcd.getNameAsString(), fileName)); Assert.assertFalse(mobTableDirExist(tableName)); }
HColumnDescriptor[] families = beforehtd.getColumnFamilies(); for (int i = 0; i < families.length; i++) { assertTrue(families[i].getNameAsString().equals("cf" + (i + 1))); assertEquals(2, afterhtd.getColumnFamilyCount()); HColumnDescriptor[] newFamilies = afterhtd.getColumnFamilies(); assertTrue(newFamilies[0].getNameAsString().equals("cf1")); assertTrue(newFamilies[1].getNameAsString().equals("cf3"));
MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, hcd1.getNameAsString()); List<Path> paths = new ArrayList<>(); if (fs.exists(mobFamilyPath)) {
Boolean foundCF = false; for (int i = 0; i < families.length; i++) { if (families[i].getNameAsString().equals(cfToDelete)) { foundCF = true; break;
copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); List<HStoreFile> sfs = new ArrayList<>(); int numDelfiles = 0;
TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString()); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); FileStatus[] files = fs.listStatus(mobFamilyPath);
@Test public void testDeleteNonMobTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = createTableDescriptor(tableName, false); HColumnDescriptor hcd = htd.getFamily(FAMILY); Table table = createTableWithOneFile(htd); try { // the mob file doesn't exist Assert.assertEquals(0, countMobFiles(tableName, hcd.getNameAsString())); Assert.assertEquals(0, countArchiveMobFiles(tableName, hcd.getNameAsString())); Assert.assertFalse(mobTableDirExist(tableName)); } finally { table.close(); TEST_UTIL.deleteTable(tableName); } Assert.assertFalse(TEST_UTIL.getAdmin().tableExists(tableName)); Assert.assertEquals(0, countMobFiles(tableName, hcd.getNameAsString())); Assert.assertEquals(0, countArchiveMobFiles(tableName, hcd.getNameAsString())); Assert.assertFalse(mobTableDirExist(tableName)); }
/** * Assert the value is store in mob. */ private static void assertIsMobReference(Cell cell, byte[] row, byte[] family, byte[] value, TableName tn) throws IOException { Assert.assertArrayEquals(row, CellUtil.cloneRow(cell)); Assert.assertArrayEquals(family, CellUtil.cloneFamily(cell)); Assert.assertFalse(Bytes.equals(value, CellUtil.cloneValue(cell))); byte[] referenceValue = CellUtil.cloneValue(cell); String fileName = MobUtils.getMobFileName(cell); int valLen = Bytes.toInt(referenceValue, 0, Bytes.SIZEOF_INT); Assert.assertEquals(value.length, valLen); Path mobFamilyPath = MobUtils.getMobFamilyPath( TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString()); Path targetPath = new Path(mobFamilyPath, fileName); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); Assert.assertTrue(fs.exists(targetPath)); } }
out.print( StringEscapeUtils.escapeHtml4(family.getNameAsString()) ); out.write("</td>\n <td>\n <table class=\"table table-striped\">\n <tr>\n <th>Property</th>\n <th>Value</th>\n </tr>\n ");
final List<Pair<byte[], String>> famPaths = new ArrayList<>(); for (HColumnDescriptor col : hdt.getColumnFamilies()) { Path hfile = new Path(dir, col.getNameAsString()); TestHRegionServerBulkLoad.createHFile(HTU.getTestFileSystem(), hfile, col.getName(), qual, val, numRows);
cf_count++; if (cf_count == 2) { region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());