private void init() { String tableName = kuduScanSpec.getTableName(); Collection<DrillbitEndpoint> endpoints = kuduStoragePlugin.getContext().getBits(); Map<String,DrillbitEndpoint> endpointMap = Maps.newHashMap(); for (DrillbitEndpoint endpoint : endpoints) { endpointMap.put(endpoint.getAddress(), endpoint); } try { List<LocatedTablet> locations = kuduStoragePlugin.getClient().openTable(tableName).getTabletsLocations(10000); for (LocatedTablet tablet : locations) { KuduWork work = new KuduWork(tablet.getPartition().getPartitionKeyStart(), tablet.getPartition().getPartitionKeyEnd()); for (Replica replica : tablet.getReplicas()) { String host = replica.getRpcHost(); DrillbitEndpoint ep = endpointMap.get(host); if (ep != null) { work.getByteMap().add(ep, DEFAULT_TABLET_SIZE); } } kuduWorkList.add(work); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public KuduInputSplit[] createInputSplits(int minNumSplits) throws IOException { startTableContext(); Preconditions.checkNotNull(tableContext,"tableContext should not be null"); List<KuduScanToken> tokens = tableContext.scanTokens(tableFilters, tableProjections, rowsLimit); KuduInputSplit[] splits = new KuduInputSplit[tokens.size()]; for (int i = 0; i < tokens.size(); i++) { KuduScanToken token = tokens.get(i); List<String> locations = new ArrayList<>(token.getTablet().getReplicas().size()); for (LocatedTablet.Replica replica : token.getTablet().getReplicas()) { locations.add(getLocation(replica.getRpcHost(), replica.getRpcPort())); } KuduInputSplit split = new KuduInputSplit( token.serialize(), i, locations.toArray(new String[locations.size()]) ); splits[i] = split; } if (splits.length < minNumSplits) { LOG.warn(" The minimum desired number of splits with your configured parallelism level " + "is {}. Current kudu splits = {}. {} instances will remain idle.", minNumSplits, splits.length, (minNumSplits - splits.length) ); } return splits; }
/** * Picks at random a tablet server that serves tablets from the passed table and restarts it. * @param table table to query for a TS to restart * @throws Exception */ public void restartTabletServer(KuduTable table) throws Exception { List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); if (tablets.isEmpty()) { fail("Table " + table.getName() + " doesn't have any tablets"); } LocatedTablet tablet = tablets.get(0); LocatedTablet.Replica replica = tablet.getReplicas().get(randomForTSRestart.nextInt(tablet.getReplicas().size())); HostAndPort hp = new HostAndPort(replica.getRpcHost(), replica.getRpcPort()); miniCluster.killTabletServer(hp); miniCluster.startTabletServer(hp); }
@Test public void testNoLeader() throws Exception { final int requestBatchSize = 10; CreateTableOptions options = getBasicCreateTableOptions(); KuduTable table = client.createTable( "testNoLeader-" + System.currentTimeMillis(), basicSchema, options); // Lookup the current locations so that we can pass some valid information to discoverTablets. List<LocatedTablet> tablets = asyncClient .locateTable(table, null, null, requestBatchSize, DEFAULT_SLEEP) .join(DEFAULT_SLEEP); LocatedTablet tablet = tablets.get(0); LocatedTablet.Replica leader = tablet.getLeaderReplica(); // Fake a master lookup that only returns one follower for the tablet. List<Master.TabletLocationsPB> tabletLocations = new ArrayList<>(); Master.TabletLocationsPB.Builder tabletPb = Master.TabletLocationsPB.newBuilder(); tabletPb.setPartition(ProtobufUtils.getFakePartitionPB()); tabletPb.setTabletId(ByteString.copyFrom(tablet.getTabletId())); tabletPb.addReplicas(ProtobufUtils.getFakeTabletReplicaPB( "master", leader.getRpcHost(), leader.getRpcPort(), Metadata.RaftPeerPB.Role.FOLLOWER)); tabletLocations.add(tabletPb.build()); try { asyncClient.discoverTablets(table, new byte[0], requestBatchSize, tabletLocations, 1000); fail("discoverTablets should throw an exception if there's no leader"); } catch (NoLeaderFoundException ex) { // Expected. } }
/** * Finds the RPC port of the given tablet's leader tserver. * @param tablet a LocatedTablet * @return the host and port of the given tablet's leader tserver * @throws Exception if we are unable to find the leader tserver */ public HostAndPort findLeaderTabletServer(LocatedTablet tablet) throws Exception { LocatedTablet.Replica leader = null; DeadlineTracker deadlineTracker = new DeadlineTracker(); deadlineTracker.setDeadline(DEFAULT_SLEEP); while (leader == null) { if (deadlineTracker.timedOut()) { fail("Timed out while trying to find a leader for this table"); } leader = tablet.getLeaderReplica(); if (leader == null) { LOG.info("Sleeping while waiting for a tablet LEADER to arise, currently slept {} ms", deadlineTracker.getElapsedMillis()); Thread.sleep(50); } } return new HostAndPort(leader.getRpcHost(), leader.getRpcPort()); }