private void init() { String tableName = kuduScanSpec.getTableName(); Collection<DrillbitEndpoint> endpoints = kuduStoragePlugin.getContext().getBits(); Map<String,DrillbitEndpoint> endpointMap = Maps.newHashMap(); for (DrillbitEndpoint endpoint : endpoints) { endpointMap.put(endpoint.getAddress(), endpoint); } try { List<LocatedTablet> locations = kuduStoragePlugin.getClient().openTable(tableName).getTabletsLocations(10000); for (LocatedTablet tablet : locations) { KuduWork work = new KuduWork(tablet.getPartition().getPartitionKeyStart(), tablet.getPartition().getPartitionKeyEnd()); for (Replica replica : tablet.getReplicas()) { String host = replica.getRpcHost(); DrillbitEndpoint ep = endpointMap.get(host); if (ep != null) { work.getByteMap().add(ep, DEFAULT_TABLET_SIZE); } } kuduWorkList.add(work); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public KuduInputSplit[] createInputSplits(int minNumSplits) throws IOException { startTableContext(); Preconditions.checkNotNull(tableContext,"tableContext should not be null"); List<KuduScanToken> tokens = tableContext.scanTokens(tableFilters, tableProjections, rowsLimit); KuduInputSplit[] splits = new KuduInputSplit[tokens.size()]; for (int i = 0; i < tokens.size(); i++) { KuduScanToken token = tokens.get(i); List<String> locations = new ArrayList<>(token.getTablet().getReplicas().size()); for (LocatedTablet.Replica replica : token.getTablet().getReplicas()) { locations.add(getLocation(replica.getRpcHost(), replica.getRpcPort())); } KuduInputSplit split = new KuduInputSplit( token.serialize(), i, locations.toArray(new String[locations.size()]) ); splits[i] = split; } if (splits.length < minNumSplits) { LOG.warn(" The minimum desired number of splits with your configured parallelism level " + "is {}. Current kudu splits = {}. {} instances will remain idle.", minNumSplits, splits.length, (minNumSplits - splits.length) ); } return splits; }
/** * Picks at random a tablet server that serves tablets from the passed table and restarts it. * @param table table to query for a TS to restart * @throws Exception */ public void restartTabletServer(KuduTable table) throws Exception { List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); if (tablets.isEmpty()) { fail("Table " + table.getName() + " doesn't have any tablets"); } LocatedTablet tablet = tablets.get(0); LocatedTablet.Replica replica = tablet.getReplicas().get(randomForTSRestart.nextInt(tablet.getReplicas().size())); HostAndPort hp = new HostAndPort(replica.getRpcHost(), replica.getRpcPort()); miniCluster.killTabletServer(hp); miniCluster.startTabletServer(hp); }
/** * Helper method to easily kill a tablet server that serves the given table's only tablet's * leader. The currently running test case will be failed if there's more than one tablet, * if the tablet has no leader after some retries, or if the tablet server was already killed. * * This method is thread-safe. * @param table a KuduTable which will get its single tablet's leader killed. * @throws Exception */ public void killTabletLeader(KuduTable table) throws Exception { List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); if (tablets.isEmpty() || tablets.size() > 1) { fail("Currently only support killing leaders for tables containing 1 tablet, table " + table.getName() + " has " + tablets.size()); } LocatedTablet tablet = tablets.get(0); if (tablet.getReplicas().size() == 1) { fail("Table " + table.getName() + " only has 1 tablet, please enable replication"); } HostAndPort hp = findLeaderTabletServer(tablet); miniCluster.killTabletServer(hp); }
private KuduTable createTableWithSplitsAndTest(String tableNamePrefix, int splitsCount) throws Exception { String newTableName = tableNamePrefix + "-" + splitsCount; CreateTableOptions builder = getBasicCreateTableOptions(); if (splitsCount != 0) { for (int i = 1; i <= splitsCount; i++) { PartialRow row = BASIC_SCHEMA.newPartialRow(); row.addInt(0, i); builder.addSplitRow(row); } } KuduTable table = client.createTable(newTableName, BASIC_SCHEMA, builder); List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP); assertEquals(splitsCount + 1, tablets.size()); assertEquals(splitsCount + 1, table.asyncGetTabletsLocations(DEFAULT_SLEEP).join().size()); for (LocatedTablet tablet : tablets) { assertEquals(3, tablet.getReplicas().size()); } return table; }