public static SpaceQuotaSnapshot toSpaceQuotaSnapshot(QuotaProtos.SpaceQuotaSnapshot proto) { return new SpaceQuotaSnapshot(SpaceQuotaStatus.toStatus(proto.getQuotaStatus()), proto.getQuotaUsage(), proto.getQuotaLimit()); }
@Override public SpaceQuotaSnapshot getTargetState( String subject, SpaceQuota spaceQuota) throws IOException { rlock.lock(); try { final long sizeLimitInBytes = spaceQuota.getSoftLimit(); long sum = 0L; for (Entry<RegionInfo,Long> entry : filterBySubject(subject)) { sum += entry.getValue(); } // Add in the size for any snapshots against this table sum += QuotaTableUtil.getNamespaceSnapshotSize(conn, subject); // Observance is defined as the size of the table being less than the limit SpaceQuotaStatus status = sum <= sizeLimitInBytes ? SpaceQuotaStatus.notInViolation() : new SpaceQuotaStatus(ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy())); return new SpaceQuotaSnapshot(status, sum, sizeLimitInBytes); } finally { rlock.unlock(); } }
@Override public SpaceQuotaSnapshot getTargetState( TableName table, SpaceQuota spaceQuota) throws IOException { rlock.lock(); try { final long sizeLimitInBytes = spaceQuota.getSoftLimit(); long sum = 0L; for (Entry<RegionInfo,Long> entry : filterBySubject(table)) { sum += entry.getValue(); } // Add in the size for any snapshots against this table sum += getSnapshotSizesForTable(table); // Observance is defined as the size of the table being less than the limit SpaceQuotaStatus status = sum <= sizeLimitInBytes ? SpaceQuotaStatus.notInViolation() : new SpaceQuotaStatus(ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy())); return new SpaceQuotaSnapshot(status, sum, sizeLimitInBytes); } finally { rlock.unlock(); } }
@Test(expected = IllegalArgumentException.class) public void testFileIsNotAFile() throws Exception { final List<String> paths = new ArrayList<>(); String path = "/1"; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(1000L); when(status.isFile()).thenReturn(false); paths.add(path); // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); policy.initialize(rss, tableName, snapshot); // If the file to bulk load isn't a file, this should throw an exception policy.computeBulkLoadSize(fs, paths); }
@Test(expected = SpaceLimitingException.class) public void testSumOfFilesOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now, but 5*1024 files would push us to violation SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); } }
@Test public void testFilesUnderLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 100L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); }
@Test(expected = SpaceLimitingException.class) public void testOneFileInBatchOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1000L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); }
@Test public void testNewPolicyOverridesOld() throws IOException { final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 512L)); final Map<TableName,SpaceQuotaSnapshot> previousPolicies = new HashMap<>(); previousPolicies.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 8192L, 512L)); // No active enforcements when(manager.getActivePoliciesAsMap()).thenReturn(previousPolicies); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); } verify(manager, never()).disableViolationPolicyEnforcement(TableName.valueOf("table1")); }
@Test public void testPoliciesAreEnforced() throws IOException { // Create a number of policies that should be enforced (usage > limit) final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 4096L, 512L)); policiesToEnforce.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 8192L, 512L)); // No active enforcements when(manager.copyQuotaSnapshots()).thenReturn(Collections.emptyMap()); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { // Ensure we enforce the policy verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); // Don't disable any policies verify(manager, never()).disableViolationPolicyEnforcement(entry.getKey()); } }
previousPolicies.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 4096L, 512L)); previousPolicies.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 8192L, 512L)); new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 256L, 512L)); policiesToEnforce.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 128L, 512L));
@Test public void testNonViolatingQuotaCachesPolicyEnforcment() { final Map<TableName,SpaceQuotaSnapshot> snapshots = new HashMap<>(); final TableName tableName = TableName.valueOf("my_table"); snapshots.put(tableName, new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024)); final ActivePolicyEnforcement ape = new ActivePolicyEnforcement( Collections.emptyMap(), snapshots, rss); SpaceViolationPolicyEnforcement policyEnforcement = ape.getPolicyEnforcement(tableName); assertTrue( "Found the wrong class: " + policyEnforcement.getClass(), policyEnforcement instanceof DefaultViolationPolicyEnforcement); SpaceViolationPolicyEnforcement copy = ape.getPolicyEnforcement(tableName); assertTrue("Expected the instance to be cached", policyEnforcement == copy); Entry<TableName,SpaceViolationPolicyEnforcement> entry = ape.getLocallyCachedPolicies().entrySet().iterator().next(); assertTrue(policyEnforcement == entry.getValue()); }
@Test public void testExceptionOnPolicyEnforcementEnable() throws Exception { final TableName tableName = TableName.valueOf("foo"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L); RegionServerServices rss = mock(RegionServerServices.class); SpaceViolationPolicyEnforcementFactory factory = mock( SpaceViolationPolicyEnforcementFactory.class); SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class); RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory); when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement); doThrow(new IOException("Failed for test!")).when(enforcement).enable(); realManager.enforceViolationPolicy(tableName, snapshot); Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements(); assertTrue("Expected active enforcements to be empty, but were " + enforcements, enforcements.isEmpty()); }
+ " reported than required."); SpaceQuotaSnapshot targetSnapshot = new SpaceQuotaSnapshot( SpaceQuotaStatus.notInViolation(), currentSnapshot.getUsage(), currentSnapshot.getLimit());
@Test public void testExceptionOnPolicyEnforcementDisable() throws Exception { final TableName tableName = TableName.valueOf("foo"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L); RegionServerServices rss = mock(RegionServerServices.class); SpaceViolationPolicyEnforcementFactory factory = mock( SpaceViolationPolicyEnforcementFactory.class); SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class); RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory); when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement); doNothing().when(enforcement).enable(); doThrow(new IOException("Failed for test!")).when(enforcement).disable(); // Enabling should work realManager.enforceViolationPolicy(tableName, snapshot); Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); // If the disable fails, we should still treat it as "active" realManager.disableViolationPolicyEnforcement(tableName); enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); } }
@Test public void testQuotaSnapshotConversion() { MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl( TEST_UTIL.getHBaseCluster().getMaster()); assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot( SpaceQuotaStatus.notInViolation(), 1024L, 2048L))); assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L))); }
@Test public void testSerDeViolationPolicies() throws Exception { final TableName tn1 = getUniqueTableName(); final SpaceQuotaSnapshot snapshot1 = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 512L, 1024L); final TableName tn2 = getUniqueTableName(); final SpaceQuotaSnapshot snapshot2 = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 512L, 1024L); final TableName tn3 = getUniqueTableName(); final SpaceQuotaSnapshot snapshot3 = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 512L, 1024L); List<Put> puts = new ArrayList<>(); puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn1, snapshot1)); puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn2, snapshot2)); puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn3, snapshot3)); final Map<TableName,SpaceQuotaSnapshot> expectedPolicies = new HashMap<>(); expectedPolicies.put(tn1, snapshot1); expectedPolicies.put(tn2, snapshot2); expectedPolicies.put(tn3, snapshot3); final Map<TableName,SpaceQuotaSnapshot> actualPolicies = new HashMap<>(); try (Table quotaTable = connection.getTable(QuotaUtil.QUOTA_TABLE_NAME)) { quotaTable.put(puts); ResultScanner scanner = quotaTable.getScanner(QuotaTableUtil.makeQuotaSnapshotScan()); for (Result r : scanner) { QuotaTableUtil.extractQuotaSnapshot(r, actualPolicies); } scanner.close(); } assertEquals(expectedPolicies, actualPolicies); }
SpaceQuotaSnapshot noInsertsSnapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 256L, 1024L); noInsertsPolicy.initialize(rss, TableName.valueOf("no_inserts"), noInsertsSnapshot); SpaceQuotaSnapshot noWritesSnapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 512L, 2048L); noWritesPolicy.initialize(rss, TableName.valueOf("no_writes"), noWritesSnapshot); SpaceQuotaSnapshot noWritesCompactionsSnapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 1024L, 4096L); noWritesCompactionsPolicy.initialize( SpaceQuotaSnapshot disableSnapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 2048L, 8192L); disablePolicy.initialize(rss, TableName.valueOf("disable"), disableSnapshot);
@Test public void testToViolation() throws Exception { final TableName tn = TableName.valueOf("inviolation"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 1024L, 512L); final Table quotaTable = mock(Table.class); when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable); final Put expectedPut = new Put(Bytes.toBytes("t." + tn.getNameAsString())); final QuotaProtos.SpaceQuotaSnapshot protoQuota = QuotaProtos.SpaceQuotaSnapshot.newBuilder() .setQuotaStatus(QuotaProtos.SpaceQuotaStatus.newBuilder().setInViolation(true) .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.NO_INSERTS)) .setQuotaLimit(512L) .setQuotaUsage(1024L) .build(); expectedPut.addColumn(Bytes.toBytes("u"), Bytes.toBytes("p"), protoQuota.toByteArray()); notifier.transitionTable(tn, snapshot); verify(quotaTable).put(argThat(new SingleCellMutationMatcher<Put>(expectedPut))); }
@Test public void testSpaceQuotaViolation() throws IOException, InterruptedException { region.getRegionServerServices().getRegionServerSpaceQuotaManager().enforceViolationPolicy(NAME, new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 10L, 100L)); Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction("test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(2, tracker.notExecutedStores.size()); tracker.notExecutedStores.sort((p1, p2) -> p1.getFirst().getColumnFamilyName() .compareTo(p2.getFirst().getColumnFamilyName())); assertEquals(Bytes.toString(CF1), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("space quota violation")); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(1).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(1).getSecond(), containsString("space quota violation")); assertTrue(tracker.beforeExecuteStores.isEmpty()); assertTrue(tracker.afterExecuteStores.isEmpty()); } }
.build(), 1024L * 256L); SpaceQuotaSnapshot tn1Snapshot = new SpaceQuotaSnapshot( SpaceQuotaStatus.notInViolation(), 1024L * 768L, 1024L * 1024L); .setEndKey(Bytes.toBytes(3)) .build(), 1024L * 256L); tn1Snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 1024L * 1024L, 1024L * 1024L); .setEndKey(Bytes.toBytes(4)) .build(), 1024L); tn1Snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L * 1024L + 1024L, 1024L * 1024L);