@Before public void removeAllQuotas() throws Exception { final Connection conn = TEST_UTIL.getConnection(); if (helper == null) { helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); } // Wait for the quota table to be created if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) { helper.waitForQuotaTable(conn); } else { // Or, clean up any quotas from previous test runs. helper.removeAllQuotas(conn); assertEquals(0, helper.listNumDefinedQuotas(conn)); } }
final Multimap<TableName, QuotaSettings> tablesWithQuotas = HashMultimap.create(); final TableName tn1 = createTable(); final TableName tn2 = createTable(); NamespaceDescriptor nd = createNamespace(); final TableName tn3 = createTableInNamespace(nd); final TableName tn4 = createTableInNamespace(nd); final TableName tn5 = createTableInNamespace(nd);
private TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception { TableName tn = helper.createTableWithRegions(10); setQuotaLimit(tn, policyToViolate, 2L); // Write more data than should be allowed and flush it to disk helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); // This should be sufficient time for the chores to run and see the change. Thread.sleep(5000); return tn; }
@Before public void removeAllQuotas() throws Exception { helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); conn = TEST_UTIL.getConnection(); admin = TEST_UTIL.getAdmin(); helper.waitForQuotaTable(conn); }
TableName createTable() throws Exception { return createTableWithRegions(1); }
@Test public void testRpcQuotaTablesAreFiltered() throws Exception { final Multimap<TableName, QuotaSettings> quotas = helper.createTablesWithSpaceQuotas(); Set<TableName> tablesWithQuotas = new HashSet<>(); Set<TableName> namespaceTablesWithQuotas = new HashSet<>(); // Partition the tables with quotas by table and ns quota helper.partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas); TableName rpcQuotaTable = helper.createTable(); TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory .throttleTable(rpcQuotaTable, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES)); // The `rpcQuotaTable` should not be included in this Set TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined(); assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables()); assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables()); }
@Test public void testNamespacesInheritSnapshotSize() throws Exception { String ns = helper.createNamespace().getName(); TableName tn = helper.createTableWithRegions(ns, 1); LOG.info("Writing data"); helper.writeData(tn, initialSize); admin.flush(tn); helper.writeData(tn, initialSize); LOG.info("Flush the table"); admin.flush(tn);
@Before public void setup() throws Exception { conn = TEST_UTIL.getConnection(); admin = TEST_UTIL.getAdmin(); helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); helper.removeAllQuotas(conn); fs = TEST_UTIL.getTestFileSystem(); conf = TEST_UTIL.getConfiguration(); }
@Test public void testGetAllTablesWithQuotas() throws Exception { final Multimap<TableName, QuotaSettings> quotas = helper.createTablesWithSpaceQuotas(); Set<TableName> tablesWithQuotas = new HashSet<>(); Set<TableName> namespaceTablesWithQuotas = new HashSet<>(); // Partition the tables with quotas by table and ns quota helper.partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas); TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined(); assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables()); assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables()); }
@Test public void testFetchSpaceQuota() throws Exception { Multimap<TableName,QuotaSettings> tables = helper.createTablesWithSpaceQuotas(); // Can pass in an empty map, we're not consulting it. chore.initializeSnapshotStores(Collections.emptyMap()); // All tables that were created should have a quota defined. for (Entry<TableName,QuotaSettings> entry : tables.entries()) { final TableName table = entry.getKey(); final QuotaSettings qs = entry.getValue(); assertTrue("QuotaSettings was an instance of " + qs.getClass(), qs instanceof SpaceLimitSettings); SpaceQuota spaceQuota = null; if (qs.getTableName() != null) { spaceQuota = chore.getTableSnapshotStore().getSpaceQuota(table); assertNotNull("Could not find table space quota for " + table, spaceQuota); } else if (qs.getNamespace() != null) { spaceQuota = chore.getNamespaceSnapshotStore().getSpaceQuota(table.getNamespaceAsString()); assertNotNull("Could not find namespace space quota for " + table.getNamespaceAsString(), spaceQuota); } else { fail("Expected table or namespace space quota"); } final SpaceLimitSettings sls = (SpaceLimitSettings) qs; assertEquals(sls.getProto().getQuota(), spaceQuota); } TableName tableWithoutQuota = helper.createTable(); assertNull(chore.getTableSnapshotStore().getSpaceQuota(tableWithoutQuota)); }
@Test public void testBulkLoading() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); ClientServiceCallable<Boolean> callable = helper.generateFileToLoad(tn, 3, 550); // Make sure the files are about as long as we expect FileSystem fs = TEST_UTIL.getTestFileSystem(); FileStatus[] files = fs.listStatus( new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files")); long totalSize = 0; for (FileStatus file : files) { assertTrue( "Expected the file, " + file.getPath() + ", length to be larger than 25KB, but was " + file.getLen(), file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE); totalSize += file.getLen(); } RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(TEST_UTIL.getConfiguration()); RpcRetryingCaller<Boolean> caller = factory.<Boolean> newCaller(); assertTrue("The bulk load failed", caller.callWithRetries(callable, Integer.MAX_VALUE)); final long finalTotalSize = totalSize; TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= finalTotalSize; } }); }
@Before public void setupForTest() throws Exception { helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); }
TableName getNextTableName() { return getNextTableName(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR); }
@Override public Void call() throws Exception { try (Connection conn = getConnection()) { final Admin admin = conn.getAdmin(); QuotaSettings qs = QuotaSettingsFactory.removeTableSpaceLimit(tn); admin.setQuota(qs); assertNull(helper.getTableSpaceQuota(conn, tn)); return null; } } });
@Test public void testNoBulkLoadsWithNoWrites() throws Exception { Put p = new Put(Bytes.toBytes("to_reject")); p.addColumn( Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); TableName tableName = writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p); // The table is now in violation. Try to do a bulk load ClientServiceCallable<Boolean> callable = helper.generateFileToLoad(tableName, 1, 50); RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(TEST_UTIL.getConfiguration()); RpcRetryingCaller<Boolean> caller = factory.newCaller(); try { caller.callWithRetries(callable, Integer.MAX_VALUE); fail("Expected the bulk load call to fail!"); } catch (SpaceLimitingException e) { // Pass LOG.trace("Caught expected exception", e); } }
TableName createTableWithRegions(int numRegions) throws Exception { return createTableWithRegions(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, numRegions); }
@Test public void testRpcQuotaTablesAreFiltered() throws Exception { final Multimap<TableName, QuotaSettings> quotas = helper.createTablesWithSpaceQuotas(); Set<TableName> tablesWithQuotas = new HashSet<>(); Set<TableName> namespaceTablesWithQuotas = new HashSet<>(); // Partition the tables with quotas by table and ns quota helper.partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas); TableName rpcQuotaTable = helper.createTable(); TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory .throttleTable(rpcQuotaTable, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES)); // The `rpcQuotaTable` should not be included in this Set TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined(); assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables()); assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables()); }
@Test public void testNamespacesInheritSnapshotSize() throws Exception { String ns = helper.createNamespace().getName(); TableName tn = helper.createTableWithRegions(ns, 1); LOG.info("Writing data"); helper.writeData(tn, initialSize); admin.flush(tn); helper.writeData(tn, initialSize); LOG.info("Flush the table"); admin.flush(tn);
@Before public void setup() throws Exception { conn = TEST_UTIL.getConnection(); admin = TEST_UTIL.getAdmin(); helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); master = TEST_UTIL.getHBaseCluster().getMaster(); helper.removeAllQuotas(conn); testChore = new SnapshotQuotaObserverChore( TEST_UTIL.getConnection(), TEST_UTIL.getConfiguration(), master.getFileSystem(), master, null); }
@Test public void testGetAllTablesWithQuotas() throws Exception { final Multimap<TableName, QuotaSettings> quotas = helper.createTablesWithSpaceQuotas(); Set<TableName> tablesWithQuotas = new HashSet<>(); Set<TableName> namespaceTablesWithQuotas = new HashSet<>(); // Partition the tables with quotas by table and ns quota helper.partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas); TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined(); assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables()); assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables()); }