private void checkCellSizeLimit(final HRegion r, final Mutation m) throws IOException { if (r.maxCellSize > 0) { CellScanner cells = m.cellScanner(); while (cells.advance()) { int size = PrivateCellUtil.estimatedSerializedSizeOf(cells.current()); if (size > r.maxCellSize) { String msg = "Cell with size " + size + " exceeds limit of " + r.maxCellSize + " bytes"; if (LOG.isDebugEnabled()) { LOG.debug(msg); } throw new DoNotRetryIOException(msg); } } } }
static Cell [] runDecoderTest(final int index, final int count, final CellScanner decoder) throws IOException { Cell [] cells = new Cell[count]; long startTime = System.currentTimeMillis(); for (int i = 0; decoder.advance(); i++) { cells[i] = decoder.current(); } LOG.info("" + index + " decoded count=" + cells.length + " in " + (System.currentTimeMillis() - startTime) + "ms for decoder " + decoder); // Ensure we did not have to grow the backing buffer. assertTrue(cells.length == count); return cells; }
/** * Fetches any persisted HBase snapshot sizes stored in the quota table. The sizes here are * computed relative to the table which the snapshot was created from. A snapshot's size will * not include the size of files which the table still refers. These sizes, in bytes, are what * is used internally to compute quota violation for tables and namespaces. * * @return A map of snapshot name to size in bytes per space quota computations */ public static Map<String,Long> getObservedSnapshotSizes(Connection conn) throws IOException { try (Table quotaTable = conn.getTable(QUOTA_TABLE_NAME); ResultScanner rs = quotaTable.getScanner(createScanForSpaceSnapshotSizes())) { final Map<String,Long> snapshotSizes = new HashMap<>(); for (Result r : rs) { CellScanner cs = r.cellScanner(); while (cs.advance()) { Cell c = cs.current(); final String snapshot = extractSnapshotNameFromSizeCell(c); final long size = parseSnapshotSize(c); snapshotSizes.put(snapshot, size); } } return snapshotSizes; } }
private void verifyRecordPresentInQuotaTable(ThrottleType type, long limit, TimeUnit tu) throws Exception { // Verify the RPC Quotas in the table try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME); ResultScanner scanner = quotaTable.getScanner(new Scan())) { Result r = Iterables.getOnlyElement(scanner); CellScanner cells = r.cellScanner(); assertTrue("Expected to find a cell", cells.advance()); assertRPCQuota(type, limit, tu, cells.current()); } }
private long extractSnapshotSize( Table quotaTable, TableName tn, String snapshot) throws IOException { Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot); Result r = quotaTable.get(g); assertNotNull(r); CellScanner cs = r.cellScanner(); assertTrue(cs.advance()); Cell c = cs.current(); assertNotNull(c); return QuotaTableUtil.extractSnapshotSize( c.getValueArray(), c.getValueOffset(), c.getValueLength()); }
private void testAdvancetHasSameRow(CellScanner scanner, final KeyValue kv) throws IOException { scanner.advance(); assertTrue(Bytes.equals(scanner.current().getRowArray(), scanner.current().getRowOffset(), scanner.current().getRowLength(), kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); } }
@Override public boolean postScannerNext(ObserverContext<RegionCoprocessorEnvironment> e, InternalScanner s, List<Result> results, int limit, boolean hasMore) throws IOException { if (checkTagPresence) { if (results.size() > 0) { // Check tag presence in the 1st cell in 1st Result Result result = results.get(0); CellScanner cellScanner = result.cellScanner(); if (cellScanner.advance()) { Cell cell = cellScanner.current(); tags = PrivateCellUtil.getTags(cell); } } } return hasMore; } }
@Override public Result preIncrement(ObserverContext<RegionCoprocessorEnvironment> e, Increment increment) throws IOException { // If authorization is not enabled, we don't care about reserved tags if (!authorizationEnabled) { return null; } for (CellScanner cellScanner = increment.cellScanner(); cellScanner.advance();) { if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { throw new FailedSanityCheckException("Increment contains cell with reserved type tag"); } } return null; }
private void verifyTableSnapshotSize( Table quotaTable, TableName tn, String snapshotName, long expectedSize) throws IOException { Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshotName)); CellScanner cs = r.cellScanner(); assertTrue(cs.advance()); Cell c = cs.current(); assertEquals(expectedSize, QuotaProtos.SpaceQuotaSnapshot.parseFrom( UnsafeByteOperations.unsafeWrap( c.getValueArray(), c.getValueOffset(), c.getValueLength())).getQuotaUsage()); assertFalse(cs.advance()); } }
@Test public void testAppendIteration() throws IOException { Append a = new Append(ROW); for (int i = 0; i < COUNT; i++) { byte [] bytes = Bytes.toBytes(i); a.addColumn(bytes, bytes, bytes); } int index = 0; for (CellScanner cellScanner = a.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); byte [] bytes = Bytes.toBytes(index++); KeyValue kv = (KeyValue)cell; assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), bytes)); assertTrue(Bytes.equals(CellUtil.cloneValue(kv), bytes)); } assertEquals(COUNT, index); }
CellScanner cs = result.cellScanner(); while (cs.advance()) { Cell current = cs.current(); try { long snapshotSize = QuotaTableUtil.parseSnapshotSize(current);
@Override public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> e, Append append) throws IOException { // If authorization is not enabled, we don't care about reserved tags if (!authorizationEnabled) { return null; } for (CellScanner cellScanner = append.cellScanner(); cellScanner.advance();) { if (!checkForReservedVisibilityTagPresence(cellScanner.current())) { throw new FailedSanityCheckException("Append contains cell with reserved type tag"); } } return null; }
@Test public void testSimpleVisibilityLabels() throws Exception { TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL, PRIVATE + "|" + CONFIDENTIAL)) { Scan s = new Scan(); s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE)); ResultScanner scanner = table.getScanner(s); Result[] next = scanner.next(3); assertTrue(next.length == 2); CellScanner cellScanner = next[0].cellScanner(); cellScanner.advance(); Cell current = cellScanner.current(); assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), row1, 0, row1.length)); cellScanner = next[1].cellScanner(); cellScanner.advance(); current = cellScanner.current(); assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), row2, 0, row2.length)); } }
@Test public void testListOfCellScannerables() throws IOException { final int count = 10; List<CellScannable> cells = new ArrayList<>(count); for (int i = 0; i < count; i++) { cells.add(createCell(i)); } HBaseRpcController controller = new HBaseRpcControllerImpl(cells); CellScanner cellScanner = controller.cellScanner(); int index = 0; for (; cellScanner.advance(); index++) { Cell cell = cellScanner.current(); byte[] indexBytes = Bytes.toBytes(index); assertTrue("" + index, Bytes.equals(indexBytes, 0, indexBytes.length, cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } assertEquals(count, index); }
void updateValueSize(final Result r) throws IOException { if (r == null || !isRandomValueSize()) return; int size = 0; for (CellScanner scanner = r.cellScanner(); scanner.advance();) { size += scanner.current().getValueLength(); } updateValueSize(size); }
Result r = scanner.cache.poll(); assertNotNull(r); CellScanner cs = r.cellScanner(); assertTrue(cs.advance()); assertEquals(kv1, cs.current()); assertFalse(cs.advance());
@Test public void testIncrementIteration() throws IOException { Increment increment = new Increment(ROW); for (int i = 0; i < COUNT; i++) { byte [] bytes = Bytes.toBytes(i); increment.addColumn(bytes, bytes, i); } int index = 0; for (CellScanner cellScanner = increment.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); int value = index; byte [] bytes = Bytes.toBytes(index++); KeyValue kv = (KeyValue)cell; assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), bytes)); long a = Bytes.toLong(CellUtil.cloneValue(kv)); assertEquals(value, a); } assertEquals(COUNT, index); }
@Test public void testResultIteration() throws IOException { Cell [] cells = new Cell[COUNT]; for(int i = 0; i < COUNT; i++) { byte [] bytes = Bytes.toBytes(i); cells[i] = new KeyValue(ROW, bytes, bytes, TIMESTAMP, bytes); } Result r = Result.create(Arrays.asList(cells)); int index = 0; for (CellScanner cellScanner = r.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); byte [] bytes = Bytes.toBytes(index++); cell.equals(new KeyValue(ROW, bytes, bytes, TIMESTAMP, bytes)); } assertEquals(COUNT, index); } }
@Override public void write(final DataOutput out) throws IOException { ProtobufUtil.toMutationNoData(MutationType.PUT, put).writeDelimitedTo(DataOutputOutputStream.from(out)); out.writeInt(put.size()); CellScanner scanner = put.cellScanner(); while(scanner.advance()) { KeyValue kv = KeyValueUtil.ensureKeyValue(scanner.current()); KeyValue.write(kv, out); } } }