AsyncTaskWorker(TaskInstance task) { this.task = task; this.callbackManager = new TaskCallbackManager(this, callbackTimer, callbackTimeoutMs, maxConcurrency, clock); Set<SystemStreamPartition> sspSet = getWorkingSSPSet(task); this.state = new AsyncTaskState(task.taskName(), task.metrics(), sspSet, task.intermediateStreams().nonEmpty()); }
private void endOfStream() { state.complete = true; try { ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); task.endOfStream(coordinator); // issue a request for shutdown of the task coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); coordinatorRequests.update(coordinator); // invoke commit on the task - if the endOfStream callback had requested a final commit. boolean needFinalCommit = coordinatorRequests.commitRequests().remove(task.taskName()); if (needFinalCommit) { task.commit(); } } finally { resume(); } }
private void init() { // schedule the timer for windowing and commiting if (task.isWindowableTask() && windowMs > 0L) { workerTimer.scheduleAtFixedRate(new Runnable() { @Override public void run() { log.trace("Task {} need window", task.taskName()); state.needWindow(); resume(); } }, windowMs, windowMs, TimeUnit.MILLISECONDS); } if (commitMs > 0L) { workerTimer.scheduleAtFixedRate(new Runnable() { @Override public void run() { log.trace("Task {} need commit", task.taskName()); state.needCommit(); resume(); } }, commitMs, commitMs, TimeUnit.MILLISECONDS); } final EpochTimeScheduler epochTimeScheduler = task.epochTimeScheduler(); if (epochTimeScheduler != null) { epochTimeScheduler.registerListener(() -> { state.needScheduler(); }); } }
/** * Returns mapping of the SystemStreamPartition to the AsyncTaskWorkers to efficiently route the envelopes */ private static Map<SystemStreamPartition, List<AsyncTaskWorker>> getSspToAsyncTaskWorkerMap( Map<TaskName, TaskInstance> taskInstances, Map<TaskName, AsyncTaskWorker> taskWorkers) { Map<SystemStreamPartition, List<AsyncTaskWorker>> sspToWorkerMap = new HashMap<>(); for (TaskInstance task : taskInstances.values()) { Set<SystemStreamPartition> ssps = JavaConverters.setAsJavaSetConverter(task.systemStreamPartitions()).asJava(); for (SystemStreamPartition ssp : ssps) { sspToWorkerMap.putIfAbsent(ssp, new ArrayList<>()); sspToWorkerMap.get(ssp).add(taskWorkers.get(task.taskName())); } } return sspToWorkerMap; }
@Override public void run() { log.trace("Task {} need commit", task.taskName()); state.needCommit(); resume(); } }, commitMs, commitMs, TimeUnit.MILLISECONDS);
@Override public void run() { try { containerMetrics.commits().inc(); long startTime = clock.nanoTime(); task.commit(); containerMetrics.commitNs().update(clock.nanoTime() - startTime); state.doneCommit(); } catch (Throwable t) { log.error("Task {} commit failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} commit completed", task.taskName()); resume(); } } };
/** * Process asynchronously. The callback needs to be fired once the processing is done. */ private void process() { final IncomingMessageEnvelope envelope = state.fetchEnvelope(); log.trace("Process ssp {} offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); final ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); TaskCallbackFactory callbackFactory = new TaskCallbackFactory() { @Override public TaskCallback createCallback() { state.startProcess(); containerMetrics.processes().inc(); return callbackManager.createCallback(task.taskName(), envelope, coordinator); } }; task.process(envelope, coordinator, callbackFactory); }
@Override public void run() { try { ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); long startTime = clock.nanoTime(); task.scheduler(coordinator); containerMetrics.timerNs().update(clock.nanoTime() - startTime); coordinatorRequests.update(coordinator); state.doneScheduler(); } catch (Throwable t) { log.error("Task {} scheduler failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} scheduler completed", task.taskName()); resume(); } } };
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
containerMetrics.windows().inc(); ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); long startTime = clock.nanoTime(); task.window(coordinator); containerMetrics.windowNs().update(clock.nanoTime() - startTime); log.error("Task {} window failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} window completed", task.taskName()); resume();
@Override public Boolean apply(TaskInstance t) { return t.isAsyncTask(); } });
@Override public void run() { log.trace("Task {} need window", task.taskName()); state.needWindow(); resume(); } }, windowMs, windowMs, TimeUnit.MILLISECONDS);
/** * Returns mapping of the SystemStreamPartition to the AsyncTaskWorkers to efficiently route the envelopes */ private static Map<SystemStreamPartition, List<AsyncTaskWorker>> getSspToAsyncTaskWorkerMap( Map<TaskName, TaskInstance> taskInstances, Map<TaskName, AsyncTaskWorker> taskWorkers) { Map<SystemStreamPartition, List<AsyncTaskWorker>> sspToWorkerMap = new HashMap<>(); for (TaskInstance task : taskInstances.values()) { Set<SystemStreamPartition> ssps = JavaConverters.setAsJavaSetConverter(task.systemStreamPartitions()).asJava(); for (SystemStreamPartition ssp : ssps) { sspToWorkerMap.putIfAbsent(ssp, new ArrayList<>()); sspToWorkerMap.get(ssp).add(taskWorkers.get(task.taskName())); } } return sspToWorkerMap; }
@Override public void run() { try { containerMetrics.commits().inc(); long startTime = clock.nanoTime(); task.commit(); containerMetrics.commitNs().update(clock.nanoTime() - startTime); state.doneCommit(); } catch (Throwable t) { log.error("Task {} commit failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} commit completed", task.taskName()); resume(); } } };
/** * Process asynchronously. The callback needs to be fired once the processing is done. */ private void process() { final IncomingMessageEnvelope envelope = state.fetchEnvelope(); log.trace("Process ssp {} offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); final ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); TaskCallbackFactory callbackFactory = new TaskCallbackFactory() { @Override public TaskCallback createCallback() { state.startProcess(); containerMetrics.processes().inc(); return callbackManager.createCallback(task.taskName(), envelope, coordinator); } }; task.process(envelope, coordinator, callbackFactory); }
@Override public void run() { try { ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); long startTime = clock.nanoTime(); task.scheduler(coordinator); containerMetrics.timerNs().update(clock.nanoTime() - startTime); coordinatorRequests.update(coordinator); state.doneScheduler(); } catch (Throwable t) { log.error("Task {} scheduler failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} scheduler completed", task.taskName()); resume(); } } };
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
containerMetrics.windows().inc(); ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); long startTime = clock.nanoTime(); task.window(coordinator); containerMetrics.windowNs().update(clock.nanoTime() - startTime); log.error("Task {} window failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} window completed", task.taskName()); resume();
@Override public Boolean apply(TaskInstance t) { return t.isAsyncTask(); } });