if (currentSnapshot.getQuotaStatus().isInViolation()) { if (LOG.isTraceEnabled()) { LOG.trace("Moving " + tableInLimbo + " out of violation because fewer region sizes were" + " reported than required."); SpaceQuotaSnapshot targetSnapshot = new SpaceQuotaSnapshot( SpaceQuotaStatus.notInViolation(), currentSnapshot.getUsage(), currentSnapshot.getLimit()); this.snapshotNotifier.transitionTable(tableInLimbo, targetSnapshot);
/** * Creates a {@link Put} to store the given {@code snapshot} for the given {@code tableName} in * the quota table. */ static Put createPutForSpaceSnapshot(TableName tableName, SpaceQuotaSnapshot snapshot) { Put p = new Put(getTableRowKey(tableName)); p.addColumn( QUOTA_FAMILY_USAGE, QUOTA_QUALIFIER_POLICY, SpaceQuotaSnapshot.toProtoSnapshot(snapshot).toByteArray()); return p; }
@Override public CompletableFuture<SpaceQuotaSnapshot> getCurrentSpaceQuotaSnapshot(String namespace) { return getCurrentSpaceQuotaSnapshot(resp -> resp.getNsSnapshotsList().stream() .filter(s -> s.getNamespace().equals(namespace)).findFirst() .map(s -> SpaceQuotaSnapshot.toSpaceQuotaSnapshot(s.getSnapshot())).orElse(null)); }
public static QuotaProtos.SpaceQuotaSnapshot toProtoSnapshot(SpaceQuotaSnapshot snapshot) { return QuotaProtos.SpaceQuotaSnapshot.newBuilder() .setQuotaStatus(SpaceQuotaStatus.toProto(snapshot.getQuotaStatus())) .setQuotaUsage(snapshot.getUsage()).setQuotaLimit(snapshot.getLimit()).build(); }
TableName table, SpaceQuotaSnapshot currentSnapshot, SpaceQuotaSnapshot targetSnapshot) throws IOException { final SpaceQuotaStatus currentStatus = currentSnapshot.getQuotaStatus(); final SpaceQuotaStatus targetStatus = targetSnapshot.getQuotaStatus(); if (!currentSnapshot.equals(targetSnapshot)) {
public static SpaceQuotaSnapshot toSpaceQuotaSnapshot(QuotaProtos.SpaceQuotaSnapshot proto) { return new SpaceQuotaSnapshot(SpaceQuotaStatus.toStatus(proto.getQuotaStatus()), proto.getQuotaUsage(), proto.getQuotaLimit()); }
/** * Checks if the given <code>snapshot</code> is in violation, allowing the snapshot to be null. * If the snapshot is null, this is interpreted as no snapshot which implies not in violation. * * @param snapshot The snapshot to operate on. * @return true if the snapshot is in violation, false otherwise. */ boolean isInViolation(SpaceQuotaSnapshot snapshot) { if (snapshot == null) { return false; } return snapshot.getQuotaStatus().isInViolation(); }
@Override public boolean evaluate() throws Exception { SpaceQuotaSnapshot snapshot = (SpaceQuotaSnapshot) conn.getAdmin() .getCurrentSpaceQuotaSnapshot(tn.getNamespaceAsString()); LOG.debug("Namespace snapshot after second ingest: " + snapshot); if (snapshot == null) { return false; } return snapshot.getUsage() > nsUsage.get() && !snapshot.getQuotaStatus().isInViolation(); } });
assertEquals(tn, entry.getKey()); final SpaceQuotaSnapshot snapshot = entry.getValue(); if (!snapshot.getQuotaStatus().isInViolation()) { LOG.info("Found a snapshot, but it was not yet in violation. " + snapshot); sleepWithInterrupt(DEFAULT_WAIT_MILLIS); final SpaceQuotaSnapshot snapshot = entry.getValue(); assertEquals("Snapshot was " + snapshot, violationPolicy, snapshot.getQuotaStatus().getPolicy().get()); assertEquals(sizeLimit, snapshot.getLimit()); assertTrue("The usage should be greater than the limit, but were " + snapshot.getUsage() + " and " + snapshot.getLimit() + ", respectively", snapshot.getUsage() > snapshot.getLimit());
@Override public boolean evaluate() throws Exception { SpaceQuotaSnapshot snapshot = (SpaceQuotaSnapshot) conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn); LOG.info("Table snapshot after initial ingest: " + snapshot); if (snapshot == null) { return false; } return snapshot.getLimit() == sizeLimit && snapshot.getUsage() > 0L; } });
@Override public SpaceQuotaSnapshot getTargetState( TableName table, SpaceQuota spaceQuota) throws IOException { rlock.lock(); try { final long sizeLimitInBytes = spaceQuota.getSoftLimit(); long sum = 0L; for (Entry<RegionInfo,Long> entry : filterBySubject(table)) { sum += entry.getValue(); } // Add in the size for any snapshots against this table sum += getSnapshotSizesForTable(table); // Observance is defined as the size of the table being less than the limit SpaceQuotaStatus status = sum <= sizeLimitInBytes ? SpaceQuotaStatus.notInViolation() : new SpaceQuotaStatus(ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy())); return new SpaceQuotaSnapshot(status, sum, sizeLimitInBytes); } finally { rlock.unlock(); } }
private int numSnapshotsInViolation(Map<TableName,SpaceQuotaSnapshot> snapshots) { int sum = 0; for (SpaceQuotaSnapshot snapshot : snapshots.values()) { if (snapshot.getQuotaStatus().isInViolation()) { sum++; } } return sum; }
@Override public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= finalSize; } });
out.print( StringUtils.byteDesc(masterSnapshot.getUsage()) ); out.write("</td>\n </tr>\n <tr>\n <td>State</td>\n <td>"); out.print( masterSnapshot.getQuotaStatus().isInViolation() ? "In Violation" : "In Observance" ); out.write("</td>\n </tr>\n");
LOG.trace(tableName + ": current=" + currentSnapshot + ", new=" + newSnapshot); if (!newSnapshot.equals(currentSnapshot)) { if (!isInViolation(currentSnapshot) && newSnapshot.getQuotaStatus().isInViolation()) { if (LOG.isTraceEnabled()) { LOG.trace("Enabling " + newSnapshot + " on " + tableName); if (isInViolation(currentSnapshot) && !newSnapshot.getQuotaStatus().isInViolation()) { if (LOG.isTraceEnabled()) { LOG.trace("Removing quota violation policy on " + tableName);
assertNotNull("Did not find snapshot for " + tn, snapshot); assertTrue( "Observed table usage was " + snapshot.getUsage(), snapshot.getUsage() >= tableSize); assertEquals(sizeLimit, snapshot.getLimit()); SpaceQuotaStatus pbStatus = snapshot.getQuotaStatus(); assertFalse(pbStatus.isInViolation());
@Override public boolean evaluate() throws Exception { SpaceQuotaSnapshot snapshot = (SpaceQuotaSnapshot) conn.getAdmin() .getCurrentSpaceQuotaSnapshot(tn.getNamespaceAsString()); LOG.debug("Namespace snapshot after initial ingest: " + snapshot); if (snapshot == null) { return false; } nsUsage.set(snapshot.getUsage()); return snapshot.getLimit() == nsLimit && snapshot.getUsage() > 0; } });
@Test(expected = IllegalArgumentException.class) public void testFileIsNotAFile() throws Exception { final List<String> paths = new ArrayList<>(); String path = "/1"; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(1000L); when(status.isFile()).thenReturn(false); paths.add(path); // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); policy.initialize(rss, tableName, snapshot); // If the file to bulk load isn't a file, this should throw an exception policy.computeBulkLoadSize(fs, paths); }