/** * Canary entry point for specified table. * @throws Exception */ private static List<Future<Void>> sniff(final Admin admin, final Sink sink, String tableName, ExecutorService executor, TaskType taskType, boolean rawScanEnabled, LongAdder readLatency) throws Exception { LOG.debug("Checking table is enabled and getting table descriptor for table {}", tableName); if (admin.isTableEnabled(TableName.valueOf(tableName))) { return Canary.sniff(admin, sink, admin.getDescriptor(TableName.valueOf(tableName)), executor, taskType, rawScanEnabled, readLatency); } else { LOG.warn("Table {} is not enabled", tableName); } return new LinkedList<>(); }
@Override public TTableDescriptor getTableDescriptor(TTableName table) throws TIOError, TException { try { TableName tableName = ThriftUtilities.tableNameFromThrift(table); TableDescriptor tableDescriptor = connectionCache.getAdmin().getDescriptor(tableName); return tableDescriptorFromHBase(tableDescriptor); } catch (IOException e) { throw getTIOError(e); } }
@Test public void testPartialSuccess() throws IOException, InterruptedException, ExecutionException { Admin admin = TEST_UTIL.getAdmin(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(admin.getDescriptor(TABLE_NAME)) .setCoprocessor(ErrorInjectObserver.class.getName()).build(); admin.modifyTable(htd); AsyncTable<?> table = tableGetter.apply(TABLE_NAME); table.putAll(Arrays.asList(SPLIT_KEYS).stream().map(k -> new Put(k).addColumn(FAMILY, CQ, k)) .collect(Collectors.toList())).get(); List<CompletableFuture<Result>> futures = table .get(Arrays.asList(SPLIT_KEYS).stream().map(k -> new Get(k)).collect(Collectors.toList())); for (int i = 0; i < SPLIT_KEYS.length - 1; i++) { assertArrayEquals(SPLIT_KEYS[i], futures.get(i).get().getValue(FAMILY, CQ)); } try { futures.get(SPLIT_KEYS.length - 1).get(); } catch (ExecutionException e) { assertThat(e.getCause(), instanceOf(RetriesExhaustedException.class)); } } }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getAdmin(); LOG.info("Performing action: Change split policy of table " + tableName); TableDescriptor tableDescriptor = admin.getDescriptor(tableName); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); String chosenPolicy = possiblePolicies[random.nextInt(possiblePolicies.length)]; builder.setRegionSplitPolicyClassName(chosenPolicy); LOG.info("Changing " + tableName + " split policy to " + chosenPolicy); admin.modifyTable(builder.build()); } }
/** * Modify table {@code getTableName()} to carry {@link SlowMeCoproScanOperations}. */ private void installSlowingCoproc() throws IOException, InterruptedException { int replicaCount = conf.getInt(NUM_REPLICA_COUNT_KEY, NUM_REPLICA_COUNT_DEFAULT); if (replicaCount == NUM_REPLICA_COUNT_DEFAULT) return; TableName t = getTablename(); Admin admin = util.getAdmin(); TableDescriptor desc = admin.getDescriptor(t); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(desc); builder.setCoprocessor(SlowMeCoproScanOperations.class.getName()); HBaseTestingUtility.modifyTableSync(admin, builder.build()); }
/** * Checks that all columns have the expected value and that there is the expected number of rows. * @throws IOException */ void assertExpectedTable(final Connection connection, TableName table, int count, int value) throws IOException { TableDescriptor htd = util.getAdmin().getDescriptor(table); assertNotNull(htd); try (Table t = connection.getTable(table); ResultScanner sr = t.getScanner(new Scan())) { int i = 0; for (Result r; (r = sr.next()) != null;) { r.getNoVersionMap().values().stream().flatMap(m -> m.values().stream()) .forEach(v -> assertArrayEquals(value(value), v)); i++; } assertEquals(count, i); } catch (IOException e) { fail("Failed due to exception"); } } }
@Override public void perform() throws Exception { TableDescriptor tableDescriptor = admin.getDescriptor(tableName); ColumnFamilyDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); if (columnDescriptors.length <= (protectedColumns == null ? 1 : protectedColumns.size())) { return; } int index = random.nextInt(columnDescriptors.length); while(protectedColumns != null && protectedColumns.contains(columnDescriptors[index].getNameAsString())) { index = random.nextInt(columnDescriptors.length); } byte[] colDescName = columnDescriptors[index].getName(); LOG.debug("Performing action: Removing " + Bytes.toString(colDescName)+ " from " + tableName.getNameAsString()); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); builder.removeColumnFamily(colDescName); // Don't try the modify if we're stopping if (context.isStopping()) { return; } admin.modifyTable(builder.build()); } }
/** * Checks that all columns have the expected value and that there is the expected number of rows. * @throws IOException */ void assertExpectedTable(TableName table, int count, int value) throws IOException { TableDescriptor htd = util.getAdmin().getDescriptor(table); assertNotNull(htd); try (Table t = util.getConnection().getTable(table); ResultScanner sr = t.getScanner(new Scan())) { int i = 0; for (Result r; (r = sr.next()) != null;) { r.getNoVersionMap().values().stream().flatMap(m -> m.values().stream()) .forEach(v -> assertArrayEquals(value(value), v)); i++; } assertEquals(count, i); } catch (IOException e) { fail("Failed due to exception"); } }
private void createTableIfMissing(Admin admin, boolean useMob) throws IOException { if (!admin.tableExists(TABLE_NAME)) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME); Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of) .forEachOrdered(builder::setColumnFamily); admin.createTable(builder.build()); } ColumnFamilyDescriptor cfd = admin.getDescriptor(TABLE_NAME).getColumnFamilies()[0]; if (cfd.isMobEnabled() != useMob) { admin.modifyColumnFamily(TABLE_NAME, ColumnFamilyDescriptorBuilder.newBuilder(cfd) .setMobEnabled(useMob).setMobThreshold(4).build()); } }
@Test public void testMasterObserverToModifyTableSchema() throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLENAME); for (int i = 1; i <= 3; i++) { builder.setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf" + i)).setMaxVersions(i) .build()); } try (Admin admin = UTIL.getAdmin()) { admin.createTable(builder.build()); assertOneVersion(admin.getDescriptor(TABLENAME)); builder.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf1")) .setMaxVersions(Integer.MAX_VALUE).build()); admin.modifyTable(builder.build()); assertOneVersion(admin.getDescriptor(TABLENAME)); } }
@Override public void perform() throws Exception { TableDescriptor tableDescriptor = admin.getDescriptor(tableName); ColumnFamilyDescriptor columnDescriptor = null; while (columnDescriptor == null || tableDescriptor.getColumnFamily(columnDescriptor.getName()) != null) { columnDescriptor = ColumnFamilyDescriptorBuilder.of(RandomStringUtils.randomAlphabetic(5)); } // Don't try the modify if we're stopping if (context.isStopping()) { return; } LOG.debug("Performing action: Adding " + columnDescriptor + " to " + tableName); TableDescriptor modifiedTable = TableDescriptorBuilder.newBuilder(tableDescriptor) .setColumnFamily(columnDescriptor).build(); admin.modifyTable(modifiedTable); } }
/** * Verify schema change for read only table */ @Test public void testReadOnlyTableModify() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); // Make table read only TableDescriptor htd = TableDescriptorBuilder.newBuilder(this.admin.getDescriptor(tableName)) .setReadOnly(true).build(); admin.modifyTable(htd); // try to modify the read only table now htd = TableDescriptorBuilder.newBuilder(this.admin.getDescriptor(tableName)) .setCompactionEnabled(false).build(); admin.modifyTable(htd); // Delete the table this.admin.disableTable(tableName); this.admin.deleteTable(tableName); assertFalse(this.admin.tableExists(tableName)); }
@Test public void testModifyTableOnTableWithRegionReplicas() throws Exception { TableName tableName = TableName.valueOf(name.getMethodName()); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))) .setRegionReplication(5) .build(); admin.createTable(desc); int maxFileSize = 10000000; TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc) .setMaxFileSize(maxFileSize) .build(); admin.modifyTable(newDesc); TableDescriptor newTableDesc = admin.getDescriptor(tableName); assertEquals(maxFileSize, newTableDesc.getMaxFileSize()); } }
/** * Apply a transform to all columns in a given table. If there are no columns in a table or if the context is stopping does nothing. * @param tableName the table to modify * @param transform the modification to perform. Callers will have the column name as a string and a column family builder available to them */ protected void modifyAllTableColumns(TableName tableName, BiConsumer<String, ColumnFamilyDescriptorBuilder> transform) throws IOException { HBaseTestingUtility util = this.context.getHBaseIntegrationTestingUtility(); Admin admin = util.getAdmin(); TableDescriptor tableDescriptor = admin.getDescriptor(tableName); ColumnFamilyDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); if (columnDescriptors == null || columnDescriptors.length == 0) { return; } TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); for (ColumnFamilyDescriptor descriptor : columnDescriptors) { ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(descriptor); transform.accept(descriptor.getNameAsString(), cfd); builder.modifyColumnFamily(cfd.build()); } // Don't try the modify if we're stopping if (this.context.isStopping()) { return; } admin.modifyTable(builder.build()); }
private static void setupTables() throws IOException { // Get the table name. tableName = TableName.valueOf(util.getConfiguration() .get("hbase.IntegrationTestMTTR.tableName", "IntegrationTestMTTR")); loadTableName = TableName.valueOf(util.getConfiguration() .get("hbase.IntegrationTestMTTR.loadTableName", "IntegrationTestMTTRLoadTestTool")); if (util.getAdmin().tableExists(tableName)) { util.deleteTable(tableName); } if (util.getAdmin().tableExists(loadTableName)) { util.deleteTable(loadTableName); } // Create the table. If this fails then fail everything. TableDescriptor tableDescriptor = util.getAdmin().getDescriptor(tableName); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); // Make the max file size huge so that splits don't happen during the test. builder.setMaxFileSize(Long.MAX_VALUE); ColumnFamilyDescriptorBuilder colDescriptorBldr = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); colDescriptorBldr.setMaxVersions(1); builder.setColumnFamily(colDescriptorBldr.build()); util.getAdmin().createTable(builder.build()); // Setup the table for LoadTestTool int ret = loadTool.run(new String[]{"-tn", loadTableName.getNameAsString(), "-init_only"}); assertEquals("Failed to initialize LoadTestTool", 0, ret); }
@Test(expected = TableNotDisabledException.class) public void testModifyRegionReplicasEnabledTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); // Modify region replication count TableDescriptor htd = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName)) .setRegionReplication(3).build(); try { // try to modify the region replication count without disabling the table admin.modifyTable(htd); fail("Expected an exception"); } finally { // Delete the table admin.disableTable(tableName); admin.deleteTable(tableName); assertFalse(admin.tableExists(tableName)); } }
@Test public void testEnableReplicationForTableWithRegionReplica() throws Exception { TableName tn = TableName.valueOf(name.getMethodName()); TableDescriptor td = TableDescriptorBuilder.newBuilder(tn) .setRegionReplication(5) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(noRepfamName).build()) .build(); admin1.createTable(td); try { admin1.enableTableReplication(tn); td = admin1.getDescriptor(tn); for (ColumnFamilyDescriptor fam : td.getColumnFamilies()) { assertEquals(HConstants.REPLICATION_SCOPE_GLOBAL, fam.getScope()); } } finally { utility1.deleteTable(tn); utility2.deleteTable(tn); } }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getAdmin(); // Verify descriptor from master TableDescriptor htd = admin.getDescriptor(tableName); ColumnFamilyDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); TableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = td.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }
@Test public void testRollbackAndDoubleExecutionOnline() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final String familyName = "cf2"; final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); // create the table RegionInfo[] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, "cf1"); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName); TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled()) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName)).build(); // Start the Modify procedure && kill the executor long procId = procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), newTd)); int lastStep = 3; // failing before MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); // cf2 should not be present MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), tableName, regions, "cf1"); }
ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); TableDescriptor td = UTIL.getAdmin().getDescriptor(tableName); TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled())