private void setQuotaLimit(final TableName tn, SpaceViolationPolicy policy, long sizeInMBs) throws Exception { final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, policy); TEST_UTIL.getAdmin().setQuota(settings); LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit); }
@Test public void testQuotaMerging() throws IOException { TableName tn = TableName.valueOf("foo"); QuotaSettings originalSettings = QuotaSettingsFactory.limitTableSpace( tn, 1024L * 1024L, SpaceViolationPolicy.DISABLE); QuotaSettings largerSizeLimit = QuotaSettingsFactory.limitTableSpace( tn, 5L * 1024L * 1024L, SpaceViolationPolicy.DISABLE); QuotaSettings differentPolicy = QuotaSettingsFactory.limitTableSpace( tn, 1024L * 1024L, SpaceViolationPolicy.NO_WRITES); QuotaSettings incompatibleSettings = QuotaSettingsFactory.limitNamespaceSpace( "ns1", 5L * 1024L * 1024L, SpaceViolationPolicy.NO_WRITES); assertEquals(originalSettings.merge(largerSizeLimit), largerSizeLimit); assertEquals(originalSettings.merge(differentPolicy), differentPolicy); try { originalSettings.merge(incompatibleSettings); fail("Should not be able to merge a Table space quota with a namespace space quota."); } catch (IllegalArgumentException e) { //pass } } }
@Override public TableName call() throws Exception { try (Connection conn = getConnection()) { final Admin admin = conn.getAdmin(); final TableName tn = helper.createTableWithRegions(admin, 5); final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, sizeLimit, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); admin.setQuota(settings); // Grant the normal user permission to create a table and set a quota try { AccessControlClient.grant( conn, tn, REGULARUSER_NAME, null, null, Action.READ, Action.WRITE); } catch (Throwable t) { if (t instanceof Exception) { throw (Exception) t; } throw new Exception(t); } return tn; } } });
@Override public TableName call() throws Exception { try (Connection conn = getConnection()) { Admin admin = conn.getAdmin(); final TableName tn = helper.createTableWithRegions(admin, 5); final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, sizeLimit, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); admin.setQuota(settings); // Grant the normal user permissions try { AccessControlClient.grant( conn, tn, REGULARUSER_NAME, null, null, Action.READ, Action.WRITE); } catch (Throwable t) { if (t instanceof Exception) { throw (Exception) t; } throw new Exception(t); } return tn; } } });
@Test public void testTableQuotaOverridesNamespaceQuota() throws Exception { final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS; final TableName tn = helper.createTableWithRegions(10); // 2MB limit on the table, 1GB limit on the namespace final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE; TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy)); TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitNamespaceSpace( tn.getNamespaceAsString(), namespaceLimit, policy)); // Write more data than should be allowed and flush it to disk helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); // This should be sufficient time for the chores to run and see the change. Thread.sleep(5000); // The write should be rejected because the table quota takes priority over the namespace Put p = new Put(Bytes.toBytes("to_reject")); p.addColumn( Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); verifyViolation(policy, tn, p); }
@Test public void testBulkLoading() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); ClientServiceCallable<Boolean> callable = helper.generateFileToLoad(tn, 3, 550); // Make sure the files are about as long as we expect FileSystem fs = TEST_UTIL.getTestFileSystem(); FileStatus[] files = fs.listStatus( new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files")); long totalSize = 0; for (FileStatus file : files) { assertTrue( "Expected the file, " + file.getPath() + ", length to be larger than 25KB, but was " + file.getLen(), file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE); totalSize += file.getLen(); } RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(TEST_UTIL.getConfiguration()); RpcRetryingCaller<Boolean> caller = factory.<Boolean> newCaller(); assertTrue("The bulk load failed", caller.callWithRetries(callable, Integer.MAX_VALUE)); final long finalTotalSize = totalSize; TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= finalTotalSize; } }); }
@Test public void testFlushes() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); // Write some data final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; helper.writeData(tn, initialSize); // Make sure a flush happened admin.flush(tn); // We should be able to observe the system recording an increase in size (even // though we know the filesystem scanning did not happen). TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= initialSize; } }); }
@Test public void testSnapshotsFromTables() throws Exception { TableName tn1 = helper.createTableWithRegions(1); TableName tn2 = helper.createTableWithRegions(1); TableName tn3 = helper.createTableWithRegions(1); // Set a space quota on table 1 and 2 (but not 3) admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); // Create snapshots on each table (we didn't write any data, so just skipflush) admin.snapshot(new SnapshotDescription(tn1 + "snapshot", tn1, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn2 + "snapshot", tn2, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn3 + "snapshot", tn3, SnapshotType.SKIPFLUSH)); Multimap<TableName,String> mapping = testChore.getSnapshotsToComputeSize(); assertEquals(2, mapping.size()); assertEquals(1, mapping.get(tn1).size()); assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next()); assertEquals(1, mapping.get(tn2).size()); assertEquals(tn2 + "snapshot", mapping.get(tn2).iterator().next()); admin.snapshot(new SnapshotDescription(tn2 + "snapshot1", tn2, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn3 + "snapshot1", tn3, SnapshotType.SKIPFLUSH)); mapping = testChore.getSnapshotsToComputeSize(); assertEquals(3, mapping.size()); assertEquals(1, mapping.get(tn1).size()); assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next()); assertEquals(2, mapping.get(tn2).size()); assertEquals( new HashSet<String>(Arrays.asList(tn2 + "snapshot", tn2 + "snapshot1")), mapping.get(tn2)); }
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy); TEST_UTIL.getAdmin().setQuota(settings);
QuotaSettings qs1 = QuotaSettingsFactory.limitTableSpace(tn1, sizeLimit1, violationPolicy1); tablesWithQuotas.put(tn1, qs1); admin.setQuota(qs1); QuotaSettings qs2 = QuotaSettingsFactory.limitTableSpace(tn2, sizeLimit2, violationPolicy2); tablesWithQuotas.put(tn2, qs2); admin.setQuota(qs2); QuotaSettings qs4 = QuotaSettingsFactory.limitTableSpace(tn5, sizeLimit4, violationPolicy4);
@Test public void testSpaceLimitSettings() { final TableName tableName = TableName.valueOf("foo"); final long sizeLimit = 1024L * 1024L * 1024L * 75; // 75GB final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_INSERTS; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tableName, sizeLimit, violationPolicy); assertNotNull("QuotaSettings should not be null", settings); assertTrue("Should be an instance of SpaceLimitSettings", settings instanceof SpaceLimitSettings); SpaceLimitSettings spaceLimitSettings = (SpaceLimitSettings) settings; SpaceLimitRequest protoRequest = spaceLimitSettings.getProto(); assertTrue("Request should have a SpaceQuota", protoRequest.hasQuota()); SpaceQuota quota = protoRequest.getQuota(); assertEquals(sizeLimit, quota.getSoftLimit()); assertEquals(violationPolicy, ProtobufUtil.toViolationPolicy(quota.getViolationPolicy())); assertFalse("The remove attribute should be false", quota.getRemove()); }
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy); TEST_UTIL.getAdmin().setQuota(settings);
@Test public void testTableSpaceQuotaRemoved() throws Exception { final Connection conn = TEST_UTIL.getConnection(); final Admin admin = conn.getAdmin(); final TableName tn = TableName.valueOf(testName.getMethodName()); // Drop the table if it somehow exists if (admin.tableExists(tn)) { dropTable(admin, tn); } createTable(admin, tn); assertEquals(0, getNumSpaceQuotas()); // Set space quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, 1024L, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); assertEquals(1, getNumSpaceQuotas()); // Drop the table and observe the Space quota being automatically deleted as well dropTable(admin, tn); assertEquals(0, getNumSpaceQuotas()); }
@Test public void testMajorCompaction() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); // Write some data and flush it to disk. final long sizePerBatch = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; helper.writeData(tn, sizePerBatch); admin.flush(tn); // Write the same data again, flushing it to a second file helper.writeData(tn, sizePerBatch); admin.flush(tn); // After two flushes, both hfiles would contain similar data. We should see 2x the data. TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= 2L * sizePerBatch; } }); // Rewrite the two files into one. admin.majorCompact(tn); // After we major compact the table, we should notice quickly that the amount of data in the // table is much closer to reality (the duplicate entries across the two files are removed). TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= sizePerBatch && snapshot.getUsage() <= 2L * sizePerBatch; } }); }
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS); TEST_UTIL.getAdmin().setQuota(settings);
final long sizeLimit = 1024L * 1024L * 1024L * 1024L * 5L; // 5TB final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_WRITES; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy); admin.setQuota(settings);
TableName tn = helper.createTableWithRegions(1); QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings);
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS); TEST_UTIL.getAdmin().setQuota(settings);
admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); settings = settings = QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); assertEquals(1, getNumSpaceQuotas());