/** * Create a table. * @param tableName the table name * @param families the families * @param splitKeys the split keys * @param replicaCount the replica count * @param c Configuration to use * @return A Table instance for the created table. * @throws IOException */ public Table createTable(TableName tableName, byte[][] families, byte[][] splitKeys, int replicaCount, final Configuration c) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); htd.setRegionReplication(replicaCount); return createTable(htd, families, splitKeys, c); }
protected void createTable() throws Exception { HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); htd.setRegionReplication(getNumReplicas()); UTIL.createTable(htd, new byte[][]{TEST_FAM}, null); }
private static void createTableDirectlyFromHTD(final TableName tableName) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); htd.setRegionReplication(3); // create a table with 3 replication table = HTU.createTable(htd, new byte[][] { f }, getSplits(20), new Configuration(HTU.getConfiguration())); }
/** * Setup a clean table with a certain region_replica count * * It will set tbl which needs to be closed after test * * @throws Exception */ void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception { HTableDescriptor desc = new HTableDescriptor(tablename); desc.setRegionReplication(replicaCount); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); desc.addFamily(hcd); // If a table has no CF's it doesn't get checked createTable(TEST_UTIL, desc, SPLITS); tbl = connection.getTable(tablename, tableExecutorService); List<Put> puts = new ArrayList<>(ROWKEYS.length); for (byte[] row : ROWKEYS) { Put p = new Put(row); p.addColumn(FAM, Bytes.toBytes("val"), row); puts.add(p); } tbl.put(puts); }
@Before public void before() throws Exception { Configuration conf = HTU.getConfiguration(); // Up the handlers; this test needs more than usual. conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true); conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_WAIT_FOR_PRIMARY_FLUSH_CONF_KEY, true); conf.setInt("replication.stats.thread.period.seconds", 5); conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); HTU.startMiniCluster(NB_SERVERS); htd = HTU.createTableDescriptor( name.getMethodName().substring(0, name.getMethodName().length()-3)); htd.setRegionReplication(3); HTU.getAdmin().createTable(htd); }
@Before public void setup() throws Exception { HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); htd.setRegionReplication(getNumReplicas()); UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration()); }
/** * Creates a pre-split table for load testing. If the table already exists, * logs a warning and continues. * @return the number of regions the table was split into */ public static int createPreSplitLoadTestTable(Configuration conf, TableName tableName, byte[] columnFamily, Algorithm compression, DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication, Durability durability) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); desc.setDurability(durability); desc.setRegionReplication(regionReplication); HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); hcd.setDataBlockEncoding(dataBlockEncoding); hcd.setCompressionType(compression); return createPreSplitLoadTestTable(conf, desc, hcd, numRegionsPerServer); }
/** * Creates a pre-split table for load testing. If the table already exists, * logs a warning and continues. * @return the number of regions the table was split into */ public static int createPreSplitLoadTestTable(Configuration conf, TableName tableName, byte[][] columnFamilies, Algorithm compression, DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication, Durability durability) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); desc.setDurability(durability); desc.setRegionReplication(regionReplication); HColumnDescriptor[] hcds = new HColumnDescriptor[columnFamilies.length]; for (int i = 0; i < columnFamilies.length; i++) { HColumnDescriptor hcd = new HColumnDescriptor(columnFamilies[i]); hcd.setDataBlockEncoding(dataBlockEncoding); hcd.setCompressionType(compression); hcds[i] = hcd; } return createPreSplitLoadTestTable(conf, desc, hcds, numRegionsPerServer); }
@Test public void testCreateTableWithSingleReplica() throws Exception { final int numRegions = 3; final int numReplica = 1; final TableName tableName = TableName.valueOf(name.getMethodName()); try { HTableDescriptor desc = new HTableDescriptor(tableName); desc.setRegionReplication(numReplica); desc.addFamily(new HColumnDescriptor("family")); ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions); validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection()); List<RegionInfo> hris = MetaTableAccessor.getTableRegions( ADMIN.getConnection(), tableName); assert(hris.size() == numRegions * numReplica); } finally { ADMIN.disableTable(tableName); ADMIN.deleteTable(tableName); } }
/** * Create an HTableDescriptor from provided TestOptions. */ protected static HTableDescriptor getTableDescriptor(TestOptions opts) { HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(opts.tableName)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); HColumnDescriptor familyDesc = new HColumnDescriptor(familyName); familyDesc.setDataBlockEncoding(opts.blockEncoding); familyDesc.setCompressionType(opts.compression); familyDesc.setBloomFilterType(opts.bloomType); familyDesc.setBlocksize(opts.blockSize); if (opts.inMemoryCF) { familyDesc.setInMemory(true); } familyDesc.setInMemoryCompaction(opts.inMemoryCompaction); tableDesc.addFamily(familyDesc); } if (opts.replicas != DEFAULT_OPTS.replicas) { tableDesc.setRegionReplication(opts.replicas); } if (opts.splitPolicy != null && !opts.splitPolicy.equals(DEFAULT_OPTS.splitPolicy)) { tableDesc.setRegionSplitPolicyClassName(opts.splitPolicy); } return tableDesc; }
/** * Set the number of Region replicas. */ public static void setReplicas(Admin admin, TableName table, int replicaCount) throws IOException, InterruptedException { admin.disableTable(table); HTableDescriptor desc = new HTableDescriptor(admin.getTableDescriptor(table)); desc.setRegionReplication(replicaCount); admin.modifyTable(desc.getTableName(), desc); admin.enableTable(table); }
private static void enableReplicationByModification(final TableName tableName, boolean withReplica, int initialReplicaCount, int enableReplicaCount, int splitCount) throws IOException, InterruptedException { HTableDescriptor htd = new HTableDescriptor(tableName); if (withReplica) { htd.setRegionReplication(initialReplicaCount); } if (splitCount > 0) { byte[][] splits = getSplits(splitCount); table = HTU.createTable(htd, new byte[][] { f }, splits, new Configuration(HTU.getConfiguration())); } else { table = HTU.createTable(htd, new byte[][] { f }, (byte[][]) null, new Configuration(HTU.getConfiguration())); } HBaseTestingUtility.setReplicas(HTU.getAdmin(), table.getName(), enableReplicaCount); }
@Test public void testCreateDeleteTable() throws IOException { // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable"); hdt.setRegionReplication(NB_SERVERS); hdt.addCoprocessor(SlowMeCopro.class.getName()); Table table = HTU.createTable(hdt, new byte[][]{f}, null); Put p = new Put(row); p.addColumn(f, row, row); table.put(p); Get g = new Get(row); Result r = table.get(g); Assert.assertFalse(r.isStale()); try { // But if we ask for stale we will get it SlowMeCopro.cdl.set(new CountDownLatch(1)); g = new Get(row); g.setConsistency(Consistency.TIMELINE); r = table.get(g); Assert.assertTrue(r.isStale()); SlowMeCopro.cdl.get().countDown(); } finally { SlowMeCopro.cdl.get().countDown(); SlowMeCopro.sleepTime.set(0); } HTU.getAdmin().disableTable(hdt.getTableName()); HTU.deleteTable(hdt.getTableName()); }
@Test public void testReplicaGetWithPrimaryDown() throws IOException { // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable"); hdt.setRegionReplication(NB_SERVERS); hdt.addCoprocessor(RegionServerStoppedCopro.class.getName()); try { Table table = HTU.createTable(hdt, new byte[][] { f }, null); Put p = new Put(row); p.addColumn(f, row, row); table.put(p); // Flush so it can be picked by the replica refresher thread HTU.flush(table.getName()); // Sleep for some time until data is picked up by replicas try { Thread.sleep(2 * REFRESH_PERIOD); } catch (InterruptedException e1) { LOG.error(e1.toString(), e1); } // But if we ask for stale we will get it Get g = new Get(row); g.setConsistency(Consistency.TIMELINE); Result r = table.get(g); Assert.assertTrue(r.isStale()); } finally { HTU.getAdmin().disableTable(hdt.getTableName()); HTU.deleteTable(hdt.getTableName()); } }
@Test public void testRegionReplicaReplicationForFlushAndCompaction() throws Exception { // Tests a table with region replication 3. Writes some data, and causes flushes and // compactions. Verifies that the data is readable from the replicas. Note that this // does not test whether the replicas actually pick up flushed files and apply compaction // to their stores int regionReplication = 3; final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = HTU.createTableDescriptor(tableName); htd.setRegionReplication(regionReplication); HTU.getAdmin().createTable(htd); Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(tableName); try { // load the data to the table for (int i = 0; i < 6000; i += 1000) { LOG.info("Writing data from " + i + " to " + (i+1000)); HTU.loadNumericRows(table, HBaseTestingUtility.fam1, i, i+1000); LOG.info("flushing table"); HTU.flush(tableName); LOG.info("compacting table"); HTU.compact(tableName, false); } verifyReplication(tableName, regionReplication, 0, 1000); } finally { table.close(); connection.close(); } }
String tableName = htd.getTableName().getNameAsString() + "2"; htd = HTU.createTableDescriptor(tableName); htd.setRegionReplication(regionReplication);
@Test public void testRegionReplicaWithoutMemstoreReplication() throws Exception { int regionReplication = 3; final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = HTU.createTableDescriptor(tableName); htd.setRegionReplication(regionReplication); htd.setRegionMemstoreReplication(false); HTU.getAdmin().createTable(htd); Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(tableName); try { // write data to the primary. The replicas should not receive the data final int STEP = 100; for (int i = 0; i < 3; ++i) { final int startRow = i * STEP; final int endRow = (i + 1) * STEP; LOG.info("Writing data from " + startRow + " to " + endRow); HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow); verifyReplication(tableName, regionReplication, startRow, endRow, false); // Flush the table, now the data should show up in the replicas LOG.info("flushing table"); HTU.flush(tableName); verifyReplication(tableName, regionReplication, 0, endRow, true); } } finally { table.close(); connection.close(); } }
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); htd.setRegionReplication(2); byte [] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
public void testRegionReplicaReplication(int regionReplication) throws Exception { // test region replica replication. Create a table with single region, write some data // ensure that data is replicated to the secondary region TableName tableName = TableName.valueOf("testRegionReplicaReplicationWithReplicas_" + regionReplication); HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString()); htd.setRegionReplication(regionReplication); HTU.getAdmin().createTable(htd); TableName tableNameNoReplicas = TableName.valueOf("testRegionReplicaReplicationWithReplicas_NO_REPLICAS"); HTU.deleteTableIfAny(tableNameNoReplicas); HTU.createTable(tableNameNoReplicas, HBaseTestingUtility.fam1); Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Table table = connection.getTable(tableName); Table tableNoReplicas = connection.getTable(tableNameNoReplicas); try { // load some data to the non-replicated table HTU.loadNumericRows(tableNoReplicas, HBaseTestingUtility.fam1, 6000, 7000); // load the data to the table HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000); verifyReplication(tableName, regionReplication, 0, 1000); } finally { table.close(); tableNoReplicas.close(); HTU.deleteTableIfAny(tableNameNoReplicas); connection.close(); } }
@Test public void testGetRegionLocationFromPrimaryMetaRegion() throws IOException, InterruptedException { HTU.getAdmin().setBalancerRunning(false, true); ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(true); // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor("testGetRegionLocationFromPrimaryMetaRegion"); hdt.setRegionReplication(2); try { HTU.createTable(hdt, new byte[][] { f }, null); RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = true; // Get user table location, always get it from the primary meta replica RegionLocations url = ((ClusterConnection) HTU.getConnection()) .locateRegion(hdt.getTableName(), row, false, false); } finally { RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = false; ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(false); HTU.getAdmin().setBalancerRunning(true, true); HTU.getAdmin().disableTable(hdt.getTableName()); HTU.deleteTable(hdt.getTableName()); } }