public static void addCoprocessorOnHTable(HTableDescriptor desc, Path hdfsCoprocessorJar) throws IOException { logger.info("Add coprocessor on " + desc.getNameAsString()); desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null); }
/** * Enable constraints on a table. * <p> * Currently, if you attempt to add a constraint to the table, then * Constraints will automatically be turned on. * * @param desc * table description to add the processor * @throws IOException * If the {@link ConstraintProcessor} CP couldn't be added to the * table. */ public static void enable(HTableDescriptor desc) throws IOException { // if the CP has already been loaded, do nothing String clazz = ConstraintProcessor.class.getName(); if (desc.hasCoprocessor(clazz)) { return; } // add the constrain processor CP to the table desc.addCoprocessor(clazz); }
@Test (expected=IOException.class) public void testAddCoprocessorTwice() throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); String cpName = "a.b.c.d"; htd.addCoprocessor(cpName); htd.addCoprocessor(cpName); }
private void createTable(String coprocessor) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName) .addFamily(new HColumnDescriptor(dummy)) .addFamily(new HColumnDescriptor(test)) .addCoprocessor(coprocessor); util.getAdmin().createTable(htd); }
@Test public void testCoprocessorTableEndpoint() throws Throwable { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(TEST_FAMILY)); desc.addCoprocessor(ColumnAggregationEndpoint.class.getName()); createTable(desc); verifyTable(tableName); }
/** * Creates a table with given table name and specified number of column * families if the table does not already exist. */ public void setupTable(TableName table, int cfs) throws IOException { try { LOG.info("Creating table " + table); HTableDescriptor htd = new HTableDescriptor(table); htd.addCoprocessor(MyObserver.class.getName()); MyObserver.sleepDuration = this.sleepDuration; for (int i = 0; i < 10; i++) { htd.addFamily(new HColumnDescriptor(family(i))); } UTIL.getAdmin().createTable(htd); } catch (TableExistsException tee) { LOG.info("Table " + table + " already exists"); } }
@Test public void testDynamicCoprocessorTableEndpoint() throws Throwable { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(TEST_FAMILY)); createTable(desc); desc.addCoprocessor(ColumnAggregationEndpoint.class.getName()); updateTable(desc); verifyTable(tableName); }
@Test(expected = DoNotRetryIOException.class) public void testScanOnCorruptHFile() throws IOException { TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addCoprocessor(CorruptHFileCoprocessor.class.getName()); htd.addFamily(new HColumnDescriptor(FAMILY_NAME)); Table table = TEST_UTIL.createTable(htd, null); try { loadTable(table, 1); scan(table); } finally { table.close(); } }
private void testPreBatchMutate(TableName tableName, Runnable rn)throws Exception { HTableDescriptor desc = new HTableDescriptor(tableName); desc.addCoprocessor(WaitingForScanObserver.class.getName()); desc.addFamily(new HColumnDescriptor(FAMILY)); TEST_UTIL.getAdmin().createTable(desc); ExecutorService service = Executors.newFixedThreadPool(2); service.execute(rn); final List<Cell> cells = new ArrayList<>(); service.execute(() -> { try { // waiting for update. TimeUnit.SECONDS.sleep(3); Table t = TEST_UTIL.getConnection().getTable(tableName); Scan scan = new Scan(); try (ResultScanner scanner = t.getScanner(scan)) { for (Result r : scanner) { cells.addAll(Arrays.asList(r.rawCells())); } } } catch (IOException | InterruptedException ex) { throw new RuntimeException(ex); } }); service.shutdown(); service.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); assertEquals("The write is blocking by RegionObserver#postBatchMutate" + ", so the data is invisible to reader", 0, cells.size()); TEST_UTIL.deleteTable(tableName); }
@Test public void testRegionObserverMultiCoprocessor() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName) .addFamily(new HColumnDescriptor(foo)) // add the coprocessor for the region. We add two different coprocessors .addCoprocessor(CustomRegionObserver.class.getName()) .addCoprocessor(CustomRegionObserver2.class.getName())); try (Table table = connection.getTable(tableName)) { table.get(new Get(foo)); table.get(new Get(foo)); // 2 gets } } // we will have two counters coming from two coprocs, in two different MetricRegistries assertPreGetRequestsCounter(CustomRegionObserver.class); assertPreGetRequestsCounter(CustomRegionObserver2.class); }
@Test public void testRegionObserverSingleRegion() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName) .addFamily(new HColumnDescriptor(foo)) // add the coprocessor for the region .addCoprocessor(CustomRegionObserver.class.getName())); try (Table table = connection.getTable(tableName)) { table.get(new Get(foo)); table.get(new Get(foo)); // 2 gets } } assertPreGetRequestsCounter(CustomRegionObserver.class); }
@Test public void testRegionObserverMultiTable() throws IOException { final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName1) .addFamily(new HColumnDescriptor(foo)) // add the coprocessor for the region .addCoprocessor(CustomRegionObserver.class.getName())); admin.createTable( new HTableDescriptor(tableName2) .addFamily(new HColumnDescriptor(foo)) // add the coprocessor for the region .addCoprocessor(CustomRegionObserver.class.getName())); try (Table table1 = connection.getTable(tableName1); Table table2 = connection.getTable(tableName2)) { table1.get(new Get(bar)); table2.get(new Get(foo)); // 2 gets to 2 separate tables } } assertPreGetRequestsCounter(CustomRegionObserver.class); }
/** * Test a table creation including a coprocessor path * which is on the classpath * @result Table will be created with the coprocessor */ @Test public void testCreationClasspathCoprocessor() throws Exception { Configuration conf = UTIL.getConfiguration(); // load coprocessor under test conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CoprocessorWhitelistMasterObserver.class.getName()); conf.setStrings(CoprocessorWhitelistMasterObserver.CP_COPROCESSOR_WHITELIST_PATHS_KEY, new String[]{}); // set retries low to raise exception quickly conf.setInt("hbase.client.retries.number", 5); UTIL.startMiniCluster(); HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); htd.addFamily(hcd); htd.addCoprocessor(TestRegionObserver.class.getName()); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); LOG.info("Creating Table"); admin.createTable(htd); // ensure table was created and coprocessor is added to table LOG.info("Done Creating Table"); Table t = connection.getTable(TEST_TABLE); assertEquals(1, t.getTableDescriptor().getCoprocessors().size()); } }
private void runCoprocessorConnectionToRemoteTable(Class clazz, boolean[] completeCheck) throws Throwable { // Check if given class implements RegionObserver. assert(RegionObserver.class.isAssignableFrom(clazz)); HTableDescriptor primary = new HTableDescriptor(primaryTable); primary.addFamily(new HColumnDescriptor(family)); // add our coprocessor primary.addCoprocessor(clazz.getName()); HTableDescriptor other = new HTableDescriptor(otherTable); other.addFamily(new HColumnDescriptor(family)); Admin admin = UTIL.getAdmin(); admin.createTable(primary); admin.createTable(other); Table table = UTIL.getConnection().getTable(TableName.valueOf("primary")); Put p = new Put(new byte[] { 'a' }); p.addColumn(family, null, new byte[]{'a'}); table.put(p); table.close(); Table target = UTIL.getConnection().getTable(otherTable); assertTrue("Didn't complete update to target table!", completeCheck[0]); assertEquals("Didn't find inserted row", 1, getKeyValueCount(target)); target.close(); }
/** * Test cps in the table description * @throws Exception */ @Test public void testGetSetRemoveCP() throws Exception { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); // simple CP String className = "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver"; // add and check that it is present desc.addCoprocessor(className); assertTrue(desc.hasCoprocessor(className)); // remove it and check that it is gone desc.removeCoprocessor(className); assertFalse(desc.hasCoprocessor(className)); }
@Test public void testCreateDeleteTable() throws IOException { // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable"); hdt.setRegionReplication(NB_SERVERS); hdt.addCoprocessor(SlowMeCopro.class.getName()); Table table = HTU.createTable(hdt, new byte[][]{f}, null); Put p = new Put(row); p.addColumn(f, row, row); table.put(p); Get g = new Get(row); Result r = table.get(g); Assert.assertFalse(r.isStale()); try { // But if we ask for stale we will get it SlowMeCopro.cdl.set(new CountDownLatch(1)); g = new Get(row); g.setConsistency(Consistency.TIMELINE); r = table.get(g); Assert.assertTrue(r.isStale()); SlowMeCopro.cdl.get().countDown(); } finally { SlowMeCopro.cdl.get().countDown(); SlowMeCopro.sleepTime.set(0); } HTU.getAdmin().disableTable(hdt.getTableName()); HTU.deleteTable(hdt.getTableName()); }
public Table createTable(TableName tableName, byte[][] families, int numVersions, int blockSize, String cpName) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family) .setMaxVersions(numVersions) .setBlocksize(blockSize); if (isNewVersionBehaviorEnabled()) { hcd.setNewVersionBehavior(true); } desc.addFamily(hcd); } if(cpName != null) { desc.addCoprocessor(cpName); } getAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are // assigned waitUntilAllRegionsAssigned(tableName); return getConnection().getTable(tableName); }
@Test public void testRegionObserverMultiRegion() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName) .addFamily(new HColumnDescriptor(foo)) // add the coprocessor for the region .addCoprocessor(CustomRegionObserver.class.getName()) , new byte[][]{foo}); // create with 2 regions try (Table table = connection.getTable(tableName); RegionLocator locator = connection.getRegionLocator(tableName)) { table.get(new Get(bar)); table.get(new Get(foo)); // 2 gets to 2 separate regions assertEquals(2, locator.getAllRegionLocations().size()); assertNotEquals(locator.getRegionLocation(bar).getRegionInfo(), locator.getRegionLocation(foo).getRegionInfo()); } } assertPreGetRequestsCounter(CustomRegionObserver.class); }
@BeforeClass public static void beforeClass() throws Exception { // enable store file refreshing HTU.getConfiguration().setInt( StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD); HTU.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true); HTU.getConfiguration().setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true); ConnectionUtils.setupMasterlessConnection(HTU.getConfiguration()); HTU.startMiniCluster(NB_SERVERS); // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor(TestReplicasClient.class.getSimpleName()); hdt.addCoprocessor(SlowMeCopro.class.getName()); table = HTU.createTable(hdt, new byte[][]{f}, null); try (RegionLocator locator = HTU.getConnection().getRegionLocator(hdt.getTableName())) { hriPrimary = locator.getRegionLocation(row, false).getRegionInfo(); } // mock a secondary region info to open hriSecondary = new HRegionInfo(hriPrimary.getTable(), hriPrimary.getStartKey(), hriPrimary.getEndKey(), hriPrimary.isSplit(), hriPrimary.getRegionId(), 1); // No master LOG.info("Master is going to be stopped"); TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU); Configuration c = new Configuration(HTU.getConfiguration()); c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); LOG.info("Master has stopped"); }
@Test public void testReplicaGetWithPrimaryDown() throws IOException { // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable"); hdt.setRegionReplication(NB_SERVERS); hdt.addCoprocessor(RegionServerStoppedCopro.class.getName()); try { Table table = HTU.createTable(hdt, new byte[][] { f }, null); Put p = new Put(row); p.addColumn(f, row, row); table.put(p); // Flush so it can be picked by the replica refresher thread HTU.flush(table.getName()); // Sleep for some time until data is picked up by replicas try { Thread.sleep(2 * REFRESH_PERIOD); } catch (InterruptedException e1) { LOG.error(e1.toString(), e1); } // But if we ask for stale we will get it Get g = new Get(row); g.setConsistency(Consistency.TIMELINE); Result r = table.get(g); Assert.assertTrue(r.isStale()); } finally { HTU.getAdmin().disableTable(hdt.getTableName()); HTU.deleteTable(hdt.getTableName()); } }