private SortedMap<KeyExtent,Bulk.Files> computeMappingFromFiles(FileSystem fs, Table.ID tableId, Path dirPath) throws IOException { Executor executor; ExecutorService service = null; if (this.executor != null) { executor = this.executor; } else if (numThreads > 0) { executor = service = Executors.newFixedThreadPool(numThreads); } else { String threads = context.getConfiguration().get(ClientProperty.BULK_LOAD_THREADS.getKey()); executor = service = Executors .newFixedThreadPool(ConfigurationTypeHelper.getNumThreads(threads)); } try { return computeFileToTabletMappings(fs, tableId, dirPath, executor, context); } finally { if (service != null) { service.shutdown(); } } }
List<KeyExtent> extents = findOverlappingTablets(context, extentCache, fileStatus.getPath(), fs, fileLensCache, cs); Map<KeyExtent,Long> estSizes = estimateSizes(context.getConfiguration(), fileStatus.getPath(), fileStatus.getLen(), extents, fs, fileLensCache, cs); Map<KeyExtent,Bulk.FileInfo> pathLocations = new HashMap<>();
throws AccumuloException, AccumuloSecurityException { try { long timeInMillis = context.getConfiguration().getTimeInMillis(Property.TSERV_BULK_TIMEOUT); TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context, timeInMillis);
public static List<KeyExtent> findOverlappingTablets(ClientContext context, KeyExtentCache extentCache, Path file, FileSystem fs, Cache<String,Long> fileLenCache, CryptoService cs) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder() .forFile(file.toString(), fs, fs.getConf(), cs) .withTableConfiguration(context.getConfiguration()).withFileLenCache(fileLenCache) .seekToBeginning().build()) { return findOverlappingTablets(extentCache, reader); } }
public ClientContext(SingletonReservation reservation, ClientInfo info, AccumuloConfiguration serverConf) { this.info = info; this.hadoopConf = info.getHadoopConf(); zooCache = new ZooCacheFactory().getZooCache(info.getZooKeepers(), info.getZooKeepersSessionTimeOut()); this.serverConf = serverConf; timeoutSupplier = memoizeWithExpiration( () -> getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT)); sslSupplier = memoizeWithExpiration(() -> SslConnectionParams.forClient(getConfiguration())); saslSupplier = memoizeWithExpiration( () -> SaslConnectionParams.from(getConfiguration(), getCredentials().getToken())); this.singletonReservation = Objects.requireNonNull(reservation); this.tableops = new TableOperationsImpl(this); this.namespaceops = new NamespaceOperationsImpl(this, tableops); }
private static void stopTabletServer(final ClientContext context, List<String> servers, final boolean force) throws AccumuloException, AccumuloSecurityException { if (context.getMasterLocations().size() == 0) { log.info("No masters running. Not attempting safe unload of tserver."); return; } final String zTServerRoot = getTServersZkPath(context); final ZooCache zc = context.getZooCache(); for (String server : servers) { for (int port : context.getConfiguration().getPort(Property.TSERV_CLIENTPORT)) { HostAndPort address = AddressUtil.parseAddress(server, port); final String finalServer = qualifyWithZooKeeperSessionId(zTServerRoot, zc, address.toString()); log.info("Stopping server {}", finalServer); MasterClient.executeVoid(context, client -> client.shutdownTabletServer(Tracer.traceInfo(), context.rpcCreds(), finalServer, force)); } } }
int tooManyFilesCount = 0; long sleepMillis = 100; final long maxSleepTime = context.getConfiguration() .getTimeInMillis(Property.GENERAL_MAX_SCANNER_RETRY_PERIOD);
final LogFileKey key = new LogFileKey(); final LogFileValue value = new LogFileValue(); final long memoryInBytes = context.getConfiguration() .getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY);