@SuppressWarnings("unchecked") @Before public void setup() throws IOException { conf = HBaseConfiguration.create(); rss = mock(RegionServerServices.class); manager = mock(RegionServerSpaceQuotaManager.class); conn = mock(Connection.class); when(manager.getRegionServerServices()).thenReturn(rss); when(rss.getConfiguration()).thenReturn(conf); chore = mock(SpaceQuotaRefresherChore.class); when(chore.getConnection()).thenReturn(conn); when(chore.getManager()).thenReturn(manager); doCallRealMethod().when(chore).chore(); when(chore.isInViolation(any())).thenCallRealMethod(); doCallRealMethod().when(chore).extractQuotaSnapshot(any(), any()); }
getManager().copyQuotaSnapshots(); final Map<TableName, SpaceQuotaSnapshot> newSnapshots = fetchSnapshotsFromQuotaTable(); if (LOG.isTraceEnabled()) { LOG.trace(currentSnapshots.size() + " table quota snapshots are collected, " if (!isInViolation(currentSnapshot) && newSnapshot.getQuotaStatus().isInViolation()) { if (LOG.isTraceEnabled()) { LOG.trace("Enabling " + newSnapshot + " on " + tableName); getManager().enforceViolationPolicy(tableName, newSnapshot); if (isInViolation(currentSnapshot) && !newSnapshot.getQuotaStatus().isInViolation()) { if (LOG.isTraceEnabled()) { LOG.trace("Removing quota violation policy on " + tableName); getManager().disableViolationPolicyEnforcement(tableName); LOG.trace("Removing quota violation policy on " + tableName); getManager().disableViolationPolicyEnforcement(tableName); getManager().updateQuotaSnapshot(newSnapshots); } catch (IOException e) { LOG.warn(
public synchronized void stop() { if (spaceQuotaRefresher != null) { spaceQuotaRefresher.cancel(); spaceQuotaRefresher = null; } if (regionSizeReporter != null) { regionSizeReporter.cancel(); regionSizeReporter = null; } started = false; }
/** * Reads all quota snapshots from the quota table. * * @return The current "view" of space use by each table. */ public Map<TableName, SpaceQuotaSnapshot> fetchSnapshotsFromQuotaTable() throws IOException { try (Table quotaTable = getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME); ResultScanner scanner = quotaTable.getScanner(QuotaTableUtil.makeQuotaSnapshotScan())) { Map<TableName,SpaceQuotaSnapshot> snapshots = new HashMap<>(); for (Result result : scanner) { try { extractQuotaSnapshot(result, snapshots); } catch (IllegalArgumentException e) { final String msg = "Failed to parse result for row " + Bytes.toString(result.getRow()); LOG.error(msg, e); throw new IOException(msg, e); } } return snapshots; } }
public SpaceQuotaRefresherChore(RegionServerSpaceQuotaManager manager, Connection conn) { super(SpaceQuotaRefresherChore.class.getSimpleName(), manager.getRegionServerServices(), getPeriod(manager.getRegionServerServices().getConfiguration()), getInitialDelay(manager.getRegionServerServices().getConfiguration()), getTimeUnit(manager.getRegionServerServices().getConfiguration())); this.manager = manager; this.conn = conn; }
@Test public void testNewPolicyOverridesOld() throws IOException { final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 512L)); final Map<TableName,SpaceQuotaSnapshot> previousPolicies = new HashMap<>(); previousPolicies.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 8192L, 512L)); // No active enforcements when(manager.getActivePoliciesAsMap()).thenReturn(previousPolicies); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); } verify(manager, never()).disableViolationPolicyEnforcement(TableName.valueOf("table1")); }
@Test public void testMissingDesiredColumn() throws IOException { when(chore.fetchSnapshotsFromQuotaTable()).thenCallRealMethod(); ResultScanner scanner = mock(ResultScanner.class); Table quotaTable = mock(Table.class); when(conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable); when(quotaTable.getScanner(any(Scan.class))).thenReturn(scanner); List<Result> results = new ArrayList<>(); // Give a column that isn't the one we want Cell c = new KeyValue(toBytes("t:inviolation"), toBytes("q"), toBytes("s"), new byte[0]); results.add(Result.create(Collections.singletonList(c))); when(scanner.iterator()).thenReturn(results.iterator()); try { chore.fetchSnapshotsFromQuotaTable(); fail("Expected an IOException, but did not receive one."); } catch (IOException e) { // Expected an error because we were missing the column we expected in this row. // This should only happen due to programmer error. } }
public synchronized void start() throws IOException { if (!QuotaUtil.isQuotaEnabled(rsServices.getConfiguration())) { LOG.info("Quota support disabled, not starting space quota manager."); return; } if (started) { LOG.warn("RegionServerSpaceQuotaManager has already been started!"); return; } // Start the chores this.spaceQuotaRefresher = new SpaceQuotaRefresherChore(this, rsServices.getClusterConnection()); rsServices.getChoreService().scheduleChore(spaceQuotaRefresher); this.regionSizeReporter = new RegionSizeReportingChore(rsServices); rsServices.getChoreService().scheduleChore(regionSizeReporter); // Instantiate the real RegionSizeStore this.regionSizeStore = RegionSizeStoreFactory.getInstance().createStore(); started = true; }
@Test public void testPoliciesAreEnforced() throws IOException { // Create a number of policies that should be enforced (usage > limit) final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 4096L, 512L)); policiesToEnforce.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 8192L, 512L)); // No active enforcements when(manager.copyQuotaSnapshots()).thenReturn(Collections.emptyMap()); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { // Ensure we enforce the policy verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); // Don't disable any policies verify(manager, never()).disableViolationPolicyEnforcement(entry.getKey()); } }
@Test public void testMissingAllColumns() throws IOException { when(chore.fetchSnapshotsFromQuotaTable()).thenCallRealMethod(); ResultScanner scanner = mock(ResultScanner.class); Table quotaTable = mock(Table.class); when(conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable); when(quotaTable.getScanner(any(Scan.class))).thenReturn(scanner); List<Result> results = new ArrayList<>(); results.add(Result.create(Collections.emptyList())); when(scanner.iterator()).thenReturn(results.iterator()); try { chore.fetchSnapshotsFromQuotaTable(); fail("Expected an IOException, but did not receive one."); } catch (IOException e) { // Expected an error because we had no cells in the row. // This should only happen due to programmer error. } }
@SuppressWarnings("unchecked") @Before public void setup() throws IOException { conf = HBaseConfiguration.create(); rss = mock(RegionServerServices.class); manager = mock(RegionServerSpaceQuotaManager.class); conn = mock(Connection.class); when(manager.getRegionServerServices()).thenReturn(rss); when(rss.getConfiguration()).thenReturn(conf); chore = mock(SpaceQuotaRefresherChore.class); when(chore.getConnection()).thenReturn(conn); when(chore.getManager()).thenReturn(manager); doCallRealMethod().when(chore).chore(); when(chore.isInViolation(any())).thenCallRealMethod(); doCallRealMethod().when(chore).extractQuotaSnapshot(any(), any()); }
when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore();
@Test public void testParsingError() throws IOException { when(chore.fetchSnapshotsFromQuotaTable()).thenCallRealMethod(); ResultScanner scanner = mock(ResultScanner.class); Table quotaTable = mock(Table.class); when(conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable); when(quotaTable.getScanner(any(Scan.class))).thenReturn(scanner); List<Result> results = new ArrayList<>(); Cell c = new KeyValue(toBytes("t:inviolation"), toBytes("u"), toBytes("v"), new byte[0]); results.add(Result.create(Collections.singletonList(c))); when(scanner.iterator()).thenReturn(results.iterator()); try { chore.fetchSnapshotsFromQuotaTable(); fail("Expected an IOException, but did not receive one."); } catch (IOException e) { // We provided a garbage serialized protobuf message (empty byte array), this should // in turn throw an IOException } } }
@Test public void testNewPolicyOverridesOld() throws IOException { final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 512L)); final Map<TableName,SpaceQuotaSnapshot> previousPolicies = new HashMap<>(); previousPolicies.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 8192L, 512L)); // No active enforcements when(manager.getActivePoliciesAsMap()).thenReturn(previousPolicies); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); } verify(manager, never()).disableViolationPolicyEnforcement(TableName.valueOf("table1")); }
@Test public void testMissingAllColumns() throws IOException { when(chore.fetchSnapshotsFromQuotaTable()).thenCallRealMethod(); ResultScanner scanner = mock(ResultScanner.class); Table quotaTable = mock(Table.class); when(conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable); when(quotaTable.getScanner(any(Scan.class))).thenReturn(scanner); List<Result> results = new ArrayList<>(); results.add(Result.create(Collections.emptyList())); when(scanner.iterator()).thenReturn(results.iterator()); try { chore.fetchSnapshotsFromQuotaTable(); fail("Expected an IOException, but did not receive one."); } catch (IOException e) { // Expected an error because we had no cells in the row. // This should only happen due to programmer error. } }
when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore();
@Test public void testMissingDesiredColumn() throws IOException { when(chore.fetchSnapshotsFromQuotaTable()).thenCallRealMethod(); ResultScanner scanner = mock(ResultScanner.class); Table quotaTable = mock(Table.class); when(conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable); when(quotaTable.getScanner(any(Scan.class))).thenReturn(scanner); List<Result> results = new ArrayList<>(); // Give a column that isn't the one we want Cell c = new KeyValue(toBytes("t:inviolation"), toBytes("q"), toBytes("s"), new byte[0]); results.add(Result.create(Collections.singletonList(c))); when(scanner.iterator()).thenReturn(results.iterator()); try { chore.fetchSnapshotsFromQuotaTable(); fail("Expected an IOException, but did not receive one."); } catch (IOException e) { // Expected an error because we were missing the column we expected in this row. // This should only happen due to programmer error. } }
@Test public void testPoliciesAreEnforced() throws IOException { // Create a number of policies that should be enforced (usage > limit) final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 4096L, 512L)); policiesToEnforce.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 8192L, 512L)); // No active enforcements when(manager.copyQuotaSnapshots()).thenReturn(Collections.emptyMap()); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { // Ensure we enforce the policy verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); // Don't disable any policies verify(manager, never()).disableViolationPolicyEnforcement(entry.getKey()); } }
@Test public void testParsingError() throws IOException { when(chore.fetchSnapshotsFromQuotaTable()).thenCallRealMethod(); ResultScanner scanner = mock(ResultScanner.class); Table quotaTable = mock(Table.class); when(conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable); when(quotaTable.getScanner(any(Scan.class))).thenReturn(scanner); List<Result> results = new ArrayList<>(); Cell c = new KeyValue(toBytes("t:inviolation"), toBytes("u"), toBytes("v"), new byte[0]); results.add(Result.create(Collections.singletonList(c))); when(scanner.iterator()).thenReturn(results.iterator()); try { chore.fetchSnapshotsFromQuotaTable(); fail("Expected an IOException, but did not receive one."); } catch (IOException e) { // We provided a garbage serialized protobuf message (empty byte array), this should // in turn throw an IOException } } }