public HBaseRpcControllerImpl(final List<CellScannable> cellIterables) { this.cellScanner = cellIterables == null ? null : CellUtil.createCellScanner(cellIterables); }
/** * @param cellIterable * @return CellScanner interface over <code>cellIterable</code> */ public static CellScanner createCellScanner(final Iterable<Cell> cellIterable) { if (cellIterable == null) return null; return createCellScanner(cellIterable.iterator()); }
@Override public CellScanner cellScanner() { return CellUtil.createCellScanner(getFamilyCellMap()); }
private CellScanner doCreateCellScanner(final int listsCount, final int itemsPerList) throws IOException { List<CellScannable> cells = new ArrayList<>(listsCount); for (int i = 0; i < listsCount; i++) { CellScannable cs = new CellScannable() { @Override public CellScanner cellScanner() { return new TestCellScanner(itemsPerList); } }; cells.add(cs); } return CellUtil.createCellScanner(cells); }
@Override public void readFields(final DataInput in) throws IOException { ClientProtos.Result protosResult = ClientProtos.Result.parseDelimitedFrom(DataInputInputStream.from(in)); int size = in.readInt(); if(size < 0) { throw new IOException("Invalid size " + size); } Cell[] kvs = new Cell[size]; for (int i = 0; i < kvs.length; i++) { kvs[i] = KeyValue.create(in); } result = ProtobufUtil.toResult(protosResult, CellUtil.createCellScanner(kvs)); } @Override
private CellScanner doCreateCellArray(final int itemsPerList) { Cell [] cells = new Cell [itemsPerList]; for (int i = 0; i < itemsPerList; i++) { cells[i] = new TestCell(i); } return CellUtil.createCellScanner(cells); }
@Override public void readFields(final DataInput in) throws IOException { ClientProtos.MutationProto putProto = ClientProtos.MutationProto.parseDelimitedFrom(DataInputInputStream.from(in)); int size = in.readInt(); if(size < 0) { throw new IOException("Invalid size " + size); } Cell[] kvs = new Cell[size]; for (int i = 0; i < kvs.length; i++) { kvs[i] = KeyValue.create(in); } put = ProtobufUtil.toPut(putProto, CellUtil.createCellScanner(kvs)); } @Override
static CellScanner getSizedCellScanner(final Cell[] cells) { int size = -1; for (Cell cell : cells) { size += PrivateCellUtil.estimatedSerializedSizeOf(cell); } final int totalSize = ClassSize.align(size); final CellScanner cellScanner = CellUtil.createCellScanner(cells); return new SizedCellScanner() { @Override public long heapSize() { return totalSize; } @Override public Cell current() { return cellScanner.current(); } @Override public boolean advance() throws IOException { return cellScanner.advance(); } }; }
@Override public ScanResponse answer(InvocationOnMock invocation) throws Throwable { ((HBaseRpcController) invocation.getArgument(0)).setCellScanner(CellUtil .createCellScanner(cellScannables)); return builder.setScannerId(1234567890L).setMoreResults(false).build(); } });
/** * Insert a mix of puts and deletes * @throws Exception */ @Test public void testMixedPutDelete() throws Exception { List<WALEntry> entries = new ArrayList<>(BATCH_SIZE/2); List<Cell> cells = new ArrayList<>(); for(int i = 0; i < BATCH_SIZE/2; i++) { entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells), replicationClusterId, baseNamespaceDir, hfileArchiveDir); entries = new ArrayList<>(BATCH_SIZE); cells = new ArrayList<>(); for(int i = 0; i < BATCH_SIZE; i++) { entries.add(createEntry(TABLE_NAME1, i, i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); Scan scan = new Scan(); ResultScanner scanRes = table1.getScanner(scan); assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length); }
/** * Insert a whole batch of entries * @throws Exception */ @Test public void testBatchSink() throws Exception { List<WALEntry> entries = new ArrayList<>(BATCH_SIZE); List<Cell> cells = new ArrayList<>(); for(int i = 0; i < BATCH_SIZE; i++) { entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); Scan scan = new Scan(); ResultScanner scanRes = table1.getScanner(scan); assertEquals(BATCH_SIZE, scanRes.next(BATCH_SIZE).length); }
/** * Insert to 2 different tables * @throws Exception */ @Test public void testMixedPutTables() throws Exception { List<WALEntry> entries = new ArrayList<>(BATCH_SIZE/2); List<Cell> cells = new ArrayList<>(); for(int i = 0; i < BATCH_SIZE; i++) { entries.add(createEntry( i % 2 == 0 ? TABLE_NAME2 : TABLE_NAME1, i, KeyValue.Type.Put, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); Scan scan = new Scan(); ResultScanner scanRes = table2.getScanner(scan); for(Result res : scanRes) { assertTrue(Bytes.toInt(res.getRow()) % 2 == 0); } }
private void addResults(ScanResponse.Builder builder, List<Result> results, HBaseRpcController controller, boolean isDefaultRegion, boolean clientCellBlockSupported) { builder.setStale(!isDefaultRegion); if (results.isEmpty()) { return; } if (clientCellBlockSupported) { for (Result res : results) { builder.addCellsPerResult(res.size()); builder.addPartialFlagPerResult(res.mayHaveMoreCellsInRow()); } controller.setCellScanner(CellUtil.createCellScanner(results)); } else { for (Result res : results) { ClientProtos.Result pbr = ProtobufUtil.toResult(res); builder.addResults(pbr); } } }
/** * Insert then do different types of deletes * @throws Exception */ @Test public void testMixedDeletes() throws Exception { List<WALEntry> entries = new ArrayList<>(3); List<Cell> cells = new ArrayList<>(); for(int i = 0; i < 3; i++) { entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); entries = new ArrayList<>(3); cells = new ArrayList<>(); entries.add(createEntry(TABLE_NAME1, 0, KeyValue.Type.DeleteColumn, cells)); entries.add(createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily, cells)); entries.add(createEntry(TABLE_NAME1, 2, KeyValue.Type.DeleteColumn, cells)); SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); Scan scan = new Scan(); ResultScanner scanRes = table1.getScanner(scan); assertEquals(0, scanRes.next(3).length); }
/** * Puts are buffered, but this tests when a delete (not-buffered) is applied * before the actual Put that creates it. * @throws Exception */ @Test public void testApplyDeleteBeforePut() throws Exception { List<WALEntry> entries = new ArrayList<>(5); List<Cell> cells = new ArrayList<>(); for(int i = 0; i < 2; i++) { entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); } entries.add(createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily, cells)); for(int i = 3; i < 5; i++) { entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()), replicationClusterId, baseNamespaceDir, hfileArchiveDir); Get get = new Get(Bytes.toBytes(1)); Result res = table1.get(get); assertEquals(0, res.size()); }
static void doBuildCellBlockUndoCellBlock(final CellBlockBuilder builder, final Codec codec, final CompressionCodec compressor, final int count, final int size, final boolean sized) throws IOException { Cell[] cells = getCells(count, size); CellScanner cellScanner = sized ? getSizedCellScanner(cells) : CellUtil.createCellScanner(Arrays.asList(cells).iterator()); ByteBuffer bb = builder.buildCellBlock(codec, compressor, cellScanner); cellScanner = builder.createCellScannerReusingBuffers(codec, compressor, new SingleByteBuff(bb)); int i = 0; while (cellScanner.advance()) { i++; } assertEquals(count, i); }
/** * Was overflowing if 100k or so lists of cellscanners to return. */ @Test public void testCreateCellScannerOverflow() throws IOException { consume(doCreateCellScanner(1, 1), 1 * 1); consume(doCreateCellScanner(3, 0), 3 * 0); consume(doCreateCellScanner(3, 3), 3 * 3); consume(doCreateCellScanner(0, 1), 0 * 1); // Do big number. See HBASE-11813 for why. final int hundredK = 100000; consume(doCreateCellScanner(hundredK, 0), hundredK * 0); consume(doCreateCellArray(1), 1); consume(doCreateCellArray(0), 0); consume(doCreateCellArray(3), 3); List<CellScannable> cells = new ArrayList<>(hundredK); for (int i = 0; i < hundredK; i++) { cells.add(new TestCellScannable(1)); } consume(CellUtil.createCellScanner(cells), hundredK * 1); NavigableMap<byte [], List<Cell>> m = new TreeMap<>(Bytes.BYTES_COMPARATOR); List<Cell> cellArray = new ArrayList<>(hundredK); for (int i = 0; i < hundredK; i++) { cellArray.add(new TestCell(i)); } m.put(new byte [] {'f'}, cellArray); consume(CellUtil.createCellScanner(m), hundredK * 1); }
@Override public EchoResponseProto echo(RpcController controller, EchoRequestProto request) throws ServiceException { if (controller instanceof HBaseRpcController) { HBaseRpcController pcrc = (HBaseRpcController) controller; // If cells, scan them to check we are able to iterate what we were given and since this is an // echo, just put them back on the controller creating a new block. Tests our block building. CellScanner cellScanner = pcrc.cellScanner(); List<Cell> list = null; if (cellScanner != null) { list = new ArrayList<>(); try { while (cellScanner.advance()) { list.add(cellScanner.current()); } } catch (IOException e) { throw new ServiceException(e); } } cellScanner = CellUtil.createCellScanner(list); pcrc.setCellScanner(cellScanner); } return EchoResponseProto.newBuilder().setMessage(request.getMessage()).build(); }
/** Tests that the rpc scheduler is called when requests arrive. */ @Test public void testRpcMaxRequestSize() throws IOException, ServiceException { Configuration conf = new Configuration(CONF); conf.setInt(RpcServer.MAX_REQUEST_SIZE, 1000); RpcServer rpcServer = createRpcServer(null, "testRpcServer", Lists.newArrayList(new RpcServer.BlockingServiceAndInterface( SERVICE, null)), new InetSocketAddress("localhost", 0), conf, new FifoRpcScheduler(conf, 1)); try (AbstractRpcClient<?> client = createRpcClient(conf)) { rpcServer.start(); BlockingInterface stub = newBlockingStub(client, rpcServer.getListenerAddress()); StringBuilder message = new StringBuilder(1200); for (int i = 0; i < 200; i++) { message.append("hello."); } // set total RPC size bigger than 100 bytes EchoRequestProto param = EchoRequestProto.newBuilder().setMessage(message.toString()).build(); stub.echo( new HBaseRpcControllerImpl(CellUtil.createCellScanner(ImmutableList.<Cell> of(CELL))), param); fail("RPC should have failed because it exceeds max request size"); } catch (ServiceException e) { LOG.info("Caught expected exception: " + e); assertTrue(e.toString(), StringUtils.stringifyException(e).contains("RequestTooBigException")); } finally { rpcServer.stop(); } }
results.add(result); ((HBaseRpcController) controller).setCellScanner(CellUtil .createCellScanner(results)); builder.setMoreResults(true);