@Override public void run() { while (!done.get()) { try { HTU.loadNumericRows(table, fam, key.get(), key.get()+1000); key.addAndGet(1000); } catch (Throwable e) { ex.compareAndSet(null, e); } } } };
public static void setUp(String regionImpl) { try { CONF.set(HConstants.REGION_IMPL, regionImpl); CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName()); cluster = HTU.startMiniCluster(NUM_RS); // Create table table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY); // this will create 2 regions spread across slaves HTU.loadNumericRows(table, FAMILY, 1, 20); HTU.flush(TABLE_NAME); } catch (Exception ex) { LOG.error("Couldn't finish setup", ex); } }
@Before public void setup() throws Exception { if (cacheType.equals("bucket")) { CONF.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); CONF.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 30); } cluster = HTU.startMiniCluster(NUM_RS); rs1 = cluster.getRegionServer(0); rs2 = cluster.getRegionServer(1); // Create table table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY); HTU.loadNumericRows(table, FAMILY, 1, 10); HTU.flush(TABLE_NAME); }
Admin admin = connection.getAdmin()) { HTU.loadNumericRows(table, fam, 0, 1000); // start with some base admin.flush(table.getName()); HTU.loadNumericRows(table, fam, 1000, 2000);
HTU.loadNumericRows(table, f, 0, 1000);
@Test public void testRegionReplicaReplicationForFlushAndCompaction() throws Exception { // Tests a table with region replication 3. Writes some data, and causes flushes and // compactions. Verifies that the data is readable from the replicas. Note that this // does not test whether the replicas actually pick up flushed files and apply compaction // to their stores int regionReplication = 3; final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = HTU.createTableDescriptor(tableName); htd.setRegionReplication(regionReplication); HTU.getAdmin().createTable(htd); Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(tableName); try { // load the data to the table for (int i = 0; i < 6000; i += 1000) { LOG.info("Writing data from " + i + " to " + (i+1000)); HTU.loadNumericRows(table, HBaseTestingUtility.fam1, i, i+1000); LOG.info("flushing table"); HTU.flush(tableName); LOG.info("compacting table"); HTU.compact(tableName, false); } verifyReplication(tableName, regionReplication, 0, 1000); } finally { table.close(); connection.close(); } }
public void testRegionReplicaReplication(int regionReplication) throws Exception { // test region replica replication. Create a table with single region, write some data // ensure that data is replicated to the secondary region TableName tableName = TableName.valueOf("testRegionReplicaReplicationWithReplicas_" + regionReplication); HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString()); htd.setRegionReplication(regionReplication); HTU.getAdmin().createTable(htd); TableName tableNameNoReplicas = TableName.valueOf("testRegionReplicaReplicationWithReplicas_NO_REPLICAS"); HTU.deleteTableIfAny(tableNameNoReplicas); HTU.createTable(tableNameNoReplicas, HBaseTestingUtility.fam1); Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(tableName); Table tableNoReplicas = connection.getTable(tableNameNoReplicas); try { // load some data to the non-replicated table HTU.loadNumericRows(tableNoReplicas, HBaseTestingUtility.fam1, 6000, 7000); // load the data to the table HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000); verifyReplication(tableName, regionReplication, 0, 1000); } finally { table.close(); tableNoReplicas.close(); HTU.deleteTableIfAny(tableNameNoReplicas); connection.close(); } }
@Test public void testGetOnTargetRegionReplica() throws Exception { try { //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); // assert that we can read back from primary Assert.assertEquals(1000, HTU.countRows(table)); // flush so that region replica can read HRegion region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName()); region.flush(true); openRegion(HTU, getRS(), hriSecondary); // try directly Get against region replica byte[] row = Bytes.toBytes(String.valueOf(42)); Get get = new Get(row); get.setConsistency(Consistency.TIMELINE); get.setReplicaId(1); Result result = table.get(get); Assert.assertArrayEquals(row, result.getValue(f, null)); } finally { HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000); closeRegion(HTU, getRS(), hriSecondary); } }
@Test public void testOpenRegionReplica() throws Exception { openRegion(HTU, getRS(), hriSecondary); try { //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); // assert that we can read back from primary Assert.assertEquals(1000, HTU.countRows(table)); } finally { HTU.deleteNumericRows(table, f, 0, 1000); closeRegion(HTU, getRS(), hriSecondary); } }
@Test public void testRegionReplicaWithoutMemstoreReplication() throws Exception { int regionReplication = 3; final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = HTU.createTableDescriptor(tableName); htd.setRegionReplication(regionReplication); htd.setRegionMemstoreReplication(false); HTU.getAdmin().createTable(htd); Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(tableName); try { // write data to the primary. The replicas should not receive the data final int STEP = 100; for (int i = 0; i < 3; ++i) { final int startRow = i * STEP; final int endRow = (i + 1) * STEP; LOG.info("Writing data from " + startRow + " to " + endRow); HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow); verifyReplication(tableName, regionReplication, startRow, endRow, false); // Flush the table, now the data should show up in the replicas LOG.info("flushing table"); HTU.flush(tableName); verifyReplication(tableName, regionReplication, 0, endRow, true); } } finally { table.close(); connection.close(); } }
private void testTruncateTable(final TableName tableName, boolean preserveSplits) throws IOException { byte[][] splitKeys = new byte[2][]; splitKeys[0] = Bytes.toBytes(4); splitKeys[1] = Bytes.toBytes(8); // Create & Fill the table Table table = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY, splitKeys); try { TEST_UTIL.loadNumericRows(table, HConstants.CATALOG_FAMILY, 0, 10); assertEquals(10, TEST_UTIL.countRows(table)); } finally { table.close(); } assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); // Truncate & Verify this.admin.disableTable(tableName); this.admin.truncateTable(tableName, preserveSplits); table = TEST_UTIL.getConnection().getTable(tableName); try { assertEquals(0, TEST_UTIL.countRows(table)); } finally { table.close(); } if (preserveSplits) { assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); } else { assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); } }
@Test public void testReplayCallable() throws Exception { // tests replaying the edits to a secondary region replica using the Callable directly openRegion(HTU, rs0, hriSecondary); ClusterConnection connection = (ClusterConnection) ConnectionFactory.createConnection(HTU.getConfiguration()); //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); Assert.assertEquals(1000, entries.size()); // replay the edits to the secondary using replay callable replicateUsingCallable(connection, entries); Region region = rs0.getRegion(hriSecondary.getEncodedName()); HTU.verifyNumericRows(region, f, 0, 1000); HTU.deleteNumericRows(table, f, 0, 1000); closeRegion(HTU, rs0, hriSecondary); connection.close(); }
@Test public void testReplayCallableWithRegionMove() throws Exception { // tests replaying the edits to a secondary region replica using the Callable directly while // the region is moved to another location.It tests handling of RME. openRegion(HTU, rs0, hriSecondary); ClusterConnection connection = (ClusterConnection) ConnectionFactory.createConnection(HTU.getConfiguration()); //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); Assert.assertEquals(1000, entries.size()); // replay the edits to the secondary using replay callable replicateUsingCallable(connection, entries); Region region = rs0.getRegion(hriSecondary.getEncodedName()); HTU.verifyNumericRows(region, f, 0, 1000); HTU.loadNumericRows(table, f, 1000, 2000); // load some more data to primary // move the secondary region from RS0 to RS1 closeRegion(HTU, rs0, hriSecondary); openRegion(HTU, rs1, hriSecondary); // replicate the new data replicateUsingCallable(connection, entries); region = rs1.getRegion(hriSecondary.getEncodedName()); // verify the new data. old data may or may not be there HTU.verifyNumericRows(region, f, 1000, 2000); HTU.deleteNumericRows(table, f, 0, 2000); closeRegion(HTU, rs1, hriSecondary); connection.close(); }
/** * Tests the case where if there is some data in the primary region, reopening the region replicas * (enable/disable table, etc) makes the region replicas readable. * @throws IOException */ @Test public void testSecondaryRegionWithNonEmptyRegion() throws IOException { // Create a new table with region replication and load some data // than disable and enable the table again and verify the data from secondary try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(htd.getTableName())) { HTU.loadNumericRows(table, fam, 0, 1000); HTU.getAdmin().disableTable(htd.getTableName()); HTU.getAdmin().enableTable(htd.getTableName()); HTU.verifyNumericRows(table, fam, 0, 1000, 1); } }
HTU.loadNumericRows(table, f, 0, 1000); HTU.loadNumericRows(table, f, 1000, 1100); region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName()); region.flush(true); HTU.loadNumericRows(table, f, 2000, 2100); region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName()); region.flush(true);
@Test public void testRegionReplicaSplitRegionAssignment() throws Exception { HTU.loadNumericRows(table, f, 0, 3);
HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000); HRegion region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName()); region.flush(true);
@Test public void testRegionReplicaGets() throws Exception { try { //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); // assert that we can read back from primary Assert.assertEquals(1000, HTU.countRows(table)); // flush so that region replica can read HRegion region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName()); region.flush(true); openRegion(HTU, getRS(), hriSecondary); // first try directly against region region = getRS().getRegion(hriSecondary.getEncodedName()); assertGet(region, 42, true); assertGetRpc(hriSecondary, 42, true); } finally { HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000); closeRegion(HTU, getRS(), hriSecondary); } }
Table table = connection.getTable(htd.getTableName())) { HTU.loadNumericRows(table, fam, 0, 1000);
try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(htd.getTableName())) { HTU.loadNumericRows(table, fam, 0, 1000);