@Override public Void call() throws Exception { try (Connection conn = getConnection()) { helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); return null; } } });
@Override public Void call() throws Exception { try (Connection conn = getConnection()) { helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); return null; } } });
void writeData(Connection conn, TableName tn, long sizeInBytes) throws IOException { writeData(tn, sizeInBytes, Bytes.toBytes("q1")); }
void writeData(TableName tn, long sizeInBytes, String qual) throws IOException { writeData(tn, sizeInBytes, Bytes.toBytes(qual)); }
void writeData(TableName tn, long sizeInBytes) throws IOException { writeData(testUtil.getConnection(), tn, sizeInBytes); }
private TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception { TableName tn = helper.createTableWithRegions(10); setQuotaLimit(tn, policyToViolate, 2L); // Write more data than should be allowed and flush it to disk helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); // This should be sufficient time for the chores to run and see the change. Thread.sleep(5000); return tn; }
@Test public void testTableQuotaOverridesNamespaceQuota() throws Exception { final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS; final TableName tn = helper.createTableWithRegions(10); // 2MB limit on the table, 1GB limit on the namespace final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE; TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy)); TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitNamespaceSpace( tn.getNamespaceAsString(), namespaceLimit, policy)); // Write more data than should be allowed and flush it to disk helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); // This should be sufficient time for the chores to run and see the change. Thread.sleep(5000); // The write should be rejected because the table quota takes priority over the namespace Put p = new Put(Bytes.toBytes("to_reject")); p.addColumn( Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); verifyViolation(policy, tn, p); }
@Test public void testMajorCompaction() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); // Write some data and flush it to disk. final long sizePerBatch = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; helper.writeData(tn, sizePerBatch); admin.flush(tn); // Write the same data again, flushing it to a second file helper.writeData(tn, sizePerBatch); admin.flush(tn); // After two flushes, both hfiles would contain similar data. We should see 2x the data. TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= 2L * sizePerBatch; } }); // Rewrite the two files into one. admin.majorCompact(tn); // After we major compact the table, we should notice quickly that the amount of data in the // table is much closer to reality (the duplicate entries across the two files are removed). TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= sizePerBatch && snapshot.getUsage() <= 2L * sizePerBatch; } }); }
@Test public void testFlushes() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); // Write some data final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; helper.writeData(tn, initialSize); // Make sure a flush happened admin.flush(tn); // We should be able to observe the system recording an increase in size (even // though we know the filesystem scanning did not happen). TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= initialSize; } }); }
@Test public void testRegionSizesFromMaster() throws Exception { final long tableSize = 1024L * 10L; // 10KB final int numRegions = 10; final TableName tn = helper.createTableWithRegions(numRegions); // Will write at least `tableSize` data helper.writeData(tn, tableSize); final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); final MasterQuotaManager quotaManager = master.getMasterQuotaManager(); // Make sure the master has all of the reports Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { Map<RegionInfo,Long> regionSizes = quotaManager.snapshotRegionSizes(); LOG.trace("Region sizes=" + regionSizes); return numRegions == countRegionsForTable(tn, regionSizes) && tableSize <= getTableSize(tn, regionSizes); } }); Map<TableName, Long> sizes = TEST_UTIL.getAdmin().getSpaceQuotaTableSizes(); Long size = sizes.get(tn); assertNotNull("No reported size for " + tn, size); assertTrue("Reported table size was " + size, size.longValue() >= tableSize); }
admin.setQuota(settings); helper.writeData(tn1, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE); admin.flush(tn1); Map<TableName,SpaceQuotaSnapshot> snapshots = snapshotNotifier.copySnapshots(); helper.writeData(tn2, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE); admin.flush(tn2); snapshots = snapshotNotifier.copySnapshots(); helper.writeData(tn3, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE); admin.flush(tn3); snapshots = snapshotNotifier.copySnapshots();
admin.setQuota(namespaceSettings); helper.writeData(tn1, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE); admin.flush(tn1); Map<TableName,SpaceQuotaSnapshot> snapshots = snapshotNotifier.copySnapshots(); helper.writeData(tn2, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE); admin.flush(tn2); snapshots = snapshotNotifier.copySnapshots();
helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
helper.writeData(tn, tableSize); } catch (RetriesExhaustedWithDetailsException | SpaceLimitingException e) {
final long numBatches = 6; for (long i = 0; i < numBatches; i++) { helper.writeData(tn, sizePerBatch); admin.flush(tn);
helper.writeData(tn, tableSize);
helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE); admin.flush(tn1);
helper.writeData(tn, sizePerBatch); admin.flush(tn);