public void start() throws Exception { initTriggers(); tezAmPool.start(); if (expirationTracker != null) { expirationTracker.start(); } if (amComm != null) { amComm.start(); } allocationManager.start(); }
/** Called once, when HS2 initializes. */ public static WorkloadManager create(String yarnQueue, HiveConf conf, WMFullResourcePlan plan) throws ExecutionException, InterruptedException { assert INSTANCE == null; // We could derive the expected number of AMs to pass in. // Note: we pass a null token here; the tokens to talk to plugin endpoints will only be // known once the AMs register, and they are different for every AM (unlike LLAP token). LlapPluginEndpointClientImpl amComm = new LlapPluginEndpointClientImpl(conf, null, -1); QueryAllocationManager qam = new GuaranteedTasksAllocator(conf, amComm); return (INSTANCE = new WorkloadManager(amComm, yarnQueue, conf, qam, plan)); }
@Override void close(boolean keepTmpDir) throws Exception { try { super.close(keepTmpDir); } finally { parent.unregisterOpenSession(this); if (expirationTracker != null) { expirationTracker.removeFromExpirationQueue(this); } } }
@Override public void run(Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws Exception { rproc = new MergeFileRecordProcessor(jobConf, getContext()); initializeAndRunProcessor(inputs, outputs); } }
static void reopenInternal( TezSessionState sessionState) throws Exception { HiveResources resources = sessionState.extractHiveResources(); // TODO: close basically resets the object to a bunch of nulls. // We should ideally not reuse the object because it's pointless and error-prone. sessionState.close(false); // Note: scratchdir is reused implicitly because the sessionId is the same. sessionState.open(resources); }
private WmTezSession createSession(HiveConf conf) { WmTezSession session = createSessionObject(TezSessionState.makeSessionId(), conf); session.setQueueName(yarnQueue); session.setDefault(); LOG.info("Created new interactive session object " + session.getSessionId()); return session; }
@Override void run() throws Exception { startAbortChecks(); while (sources[position].pushRecord()) { addRowAndMaybeCheckAbort(); } }
@Override protected void openInternal(String[] additionalFiles, boolean isAsync, LogHelper console, HiveResources resources) throws IOException, LoginException, URISyntaxException, TezException { super.openInternal(additionalFiles, isAsync, console, resources); parent.registerOpenSession(this); if (expirationTracker != null) { expirationTracker.addToExpirationQueue(this); } }
@Override public String toString() { return "sessionId=" + sessionId + ", queueName=" + queueName + ", user=" + user + ", doAs=" + doAsEnabled + ", isOpen=" + isOpen() + ", isDefault=" + defaultQueue; }
@Override public void closeAndReopenExpiredSession(TezSessionPoolSession session) throws Exception { // By definition, this session is not in use and can no longer be in use, so it only // affects the session pool. We can handle this inline. tezAmPool.replaceSession(ensureOwnedSession(session)); }
/** Note: this is not thread-safe. */ public static TezSessionPoolManager getInstance() { TezSessionPoolManager local = instance; if (local == null) { instance = local = new TezSessionPoolManager(); } return local; }
private void initTriggers() { if (triggerValidatorRunnable == null) { final long triggerValidationIntervalMs = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TRIGGER_VALIDATION_INTERVAL, TimeUnit.MILLISECONDS); TriggerActionHandler<?> triggerActionHandler = new KillMoveTriggerActionHandler(this); triggerValidatorRunnable = new PerPoolTriggerValidatorRunnable(perPoolProviders, triggerActionHandler, triggerValidationIntervalMs); startTriggerValidator(triggerValidationIntervalMs); } }
@Override public WmTezSession getSession( TezSessionState session, MappingInput input, HiveConf conf, final WmContext wmContext) throws Exception { // We want to wait for the iteration to finish and set the cluster fraction. WmTezSession state = super.getSession(session, input, conf, null); ensureWm(); return state; }
public PoolState(String fullName, int queryParallelism, double fraction, String schedulingPolicy, MetricsSystem ms) { this.fullName = fullName; // TODO: this actually calls the metrics system and getMetrics - that may be expensive. // For now it looks like it should be ok to do on WM thread. this.metrics = ms == null ? null : WmPoolMetrics.create(fullName, ms); update(queryParallelism, fraction, null, null, schedulingPolicy); }
@Override public void sendUpdateQuery(UpdateQueryRequestProto request, AmPluginNode node, UpdateRequestContext callback) { queueRequest(new SendUpdateQueryCallable(node, request, callback)); }
public static void endStart(SessionState startSs) throws CancellationException, InterruptedException { if (startSs.tezSessionState == null) { return; } startSs.tezSessionState.endOpen(); }
public SampleTezSessionState( String sessionId, TezSessionPoolSession.Manager parent, HiveConf conf) { super(sessionId, parent, (parent instanceof TezSessionPoolManager) ? ((TezSessionPoolManager)parent).getExpirationTracker() : null, conf); this.sessionId = sessionId; this.hiveConf = conf; waitForAmRegFuture = createDefaultWaitForAmRegistryFuture(); }
@Override public void returnAfterUse(TezSessionPoolSession session) throws Exception { super.returnAfterUse(session); ensureWm(); }