/** * Utility function to launch task 'taskId' in plan at the first host. * Local hosts are preferred over remote hosts. * * TODO: move this into a test/sample directory? Real frameworks have * to deal with much more stuff. */ public static Records execTask(PlanRequestResult plan, int taskId) throws RecordServiceException, IOException { if (taskId >= plan.tasks.size() || taskId < 0) { throw new RuntimeException("Invalid task id."); } Task task = plan.tasks.get(taskId); NetworkAddress host; if (task.localHosts.isEmpty()) { if (plan.hosts.isEmpty()) { throw new RuntimeException("No hosts are provided to run this task."); } host = plan.hosts.get(0); } else { host = task.localHosts.get(0); } Records records = new RecordServiceWorkerClient.Builder() .connect(host.hostname, host.port).execAndFetch(task); records.setCloseWorker(true); return records; } }
new RecordServiceWorkerClient.Builder(); int fetchSize = jobConf.getInt(ConfVars.FETCH_SIZE_CONF.name, DEFAULT_FETCH_SIZE); long memLimit = jobConf.getLong(ConfVars.MEM_LIMIT_CONF.name, -1); jobConf.getBoolean(ConfVars.WORKER_ENABLE_SERVER_LOGGING_CONF.name, false); if (fetchSize != -1) builder.setFetchSize(fetchSize); if (memLimit != -1) builder.setMemLimit(memLimit); if (limit != -1) builder.setLimit(limit); if (maxAttempts != -1) builder.setMaxAttempts(maxAttempts); if (taskSleepMs != -1 ) builder.setSleepDurationMs(taskSleepMs); if (connectionTimeoutMs != -1) builder.setConnectionTimeoutMs(connectionTimeoutMs); if (rpcTimeoutMs != -1) builder.setRpcTimeoutMs(rpcTimeoutMs); if (enableLogging) builder.setLoggingLevel(LOG); if (delegationToken != null) builder.setDelegationToken(delegationToken);
/** * Creates a RecordReaderCore to read the records for taskInfo. */ @SuppressWarnings("unchecked") public RecordReaderCore(Configuration config, Credentials credentials, TaskInfo taskInfo) throws RecordServiceException, IOException { Token<DelegationTokenIdentifier> token = (Token<DelegationTokenIdentifier>) credentials.getToken(DelegationTokenIdentifier.DELEGATION_KIND); RecordServiceWorkerClient.Builder builder = WorkerUtil.getBuilder(config, TokenUtils.toDelegationToken(token)); NetworkAddress address = WorkerUtil.getWorkerToConnectTo( taskInfo.getTask().taskId, taskInfo.getLocations(), taskInfo.getAllWorkerAddresses()); setTag(taskInfo.getTask(), config); try { worker_ = builder.connect(address.hostname, address.port); records_ = worker_.execAndFetch(taskInfo.getTask()); } finally { if (records_ == null) close(); } schema_ = new Schema(records_.getSchema()); }
/** * Creates a worker client connecting to the 'address' with previously * set options, and the caller must call close(). */ public RecordServiceWorkerClient connect(NetworkAddress address) throws RecordServiceException, IOException { return connect(address.hostname, address.port); } }
/** * Creates a worker client connecting to 'hostname'/'port' with previously * set options, and the caller must call close(). */ public RecordServiceWorkerClient connect(String hostname, int port) throws RecordServiceException, IOException { client_.connect(hostname, port); return client_; }