@Override public Set<String> getAllActiveExecutorServerHosts() { final Set<String> ports = getPrimaryServerHosts(); // include executor which were initially active and still has flows running try { for (final Pair<ExecutionReference, ExecutableFlow> running : this.executorLoader .fetchActiveFlows().values()) { final ExecutionReference ref = running.getFirst(); if (ref.getExecutor().isPresent()) { final Executor executor = ref.getExecutor().get(); ports.add(executor.getHost() + ":" + executor.getPort()); } } } catch (final ExecutorManagerException e) { logger.error("Failed to get all active executor server hosts.", e); } return ports; }
private void getActiveFlowsWithExecutorHelper( final List<Pair<ExecutableFlow, Optional<Executor>>> flows, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { flows.add(new Pair<>(ref.getSecond(), ref .getFirst().getExecutor())); } }
/** * Groups Executable flow by Executors to reduce number of REST calls. * * @return executor to list of flows map */ private Map<Optional<Executor>, List<ExecutableFlow>> getFlowToExecutorMap() { final HashMap<Optional<Executor>, List<ExecutableFlow>> exFlowMap = new HashMap<>(); try { for (final Pair<ExecutionReference, ExecutableFlow> runningFlow : this .executorLoader.fetchActiveFlows().values()) { final Optional<Executor> executor = runningFlow.getFirst().getExecutor(); List<ExecutableFlow> flows = exFlowMap.get(executor); if (flows == null) { flows = new ArrayList<>(); exFlowMap.put(executor, flows); } flows.add(runningFlow.getSecond()); } } catch (final ExecutorManagerException e) { logger.error("Failed to get flow to executor map"); } return exFlowMap; }
private void getActiveFlowsWithExecutorHelper( final List<Pair<ExecutableFlow, Optional<Executor>>> flows, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { flows.add(new Pair<>(ref.getSecond(), ref .getFirst().getExecutor())); } }
private Map<Optional<Executor>, List<ExecutableFlow>> getFlowToExecutorMap() { final HashMap<Optional<Executor>, List<ExecutableFlow>> exFlowMap = new HashMap<>(); for (final Pair<ExecutionReference, ExecutableFlow> runningFlow : this.runningExecutions.get() .values()) { final ExecutionReference ref = runningFlow.getFirst(); final ExecutableFlow flow = runningFlow.getSecond(); final Optional<Executor> executor = ref.getExecutor(); // We can set the next check time to prevent the checking of certain // flows. if (ref.getNextCheckTime() >= DateTime.now().getMillis()) { continue; } List<ExecutableFlow> flows = exFlowMap.get(executor); if (flows == null) { flows = new ArrayList<>(); exFlowMap.put(executor, flows); } flows.add(flow); } return exFlowMap; }
@Override public Set<String> getAllActiveExecutorServerHosts() { // Includes non primary server/hosts final HashSet<String> ports = new HashSet<>(); for (final Executor executor : this.activeExecutors.getAll()) { ports.add(executor.getHost() + ":" + executor.getPort()); } // include executor which were initially active and still has flows running for (final Pair<ExecutionReference, ExecutableFlow> running : this.runningExecutions.get() .values()) { final ExecutionReference ref = running.getFirst(); if (ref.getExecutor().isPresent()) { final Executor executor = ref.getExecutor().get(); ports.add(executor.getHost() + ":" + executor.getPort()); } } return ports; }
Map<String, Object> callWithReference(final ExecutionReference ref, final String action, final Pair<String, String>... params) throws ExecutorManagerException { final Executor executor = ref.getExecutor().get(); return callWithExecutionId(executor.getHost(), executor.getPort(), action, ref.getExecId(), null, params); }
Map<String, Object> callWithReferenceByUser(final ExecutionReference ref, final String action, final String user, final Pair<String, String>... params) throws ExecutorManagerException { final Executor executor = ref.getExecutor().get(); return callWithExecutionId(executor.getHost(), executor.getPort(), action, ref.getExecId(), user, params); }
private void assertFound(final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows, final ExecutableFlow flow, final boolean executorPresent) { assertThat(activeFlows.containsKey(flow.getExecutionId())).isTrue(); assertThat(activeFlows.get(flow.getExecutionId()).getFirst().getExecutor().isPresent()) .isEqualTo(executorPresent); }
@Test public void handleResultMissingExecutor() throws Exception { final FetchActiveExecutableFlows resultHandler = new FetchActiveExecutableFlows(); mockResultWithData(); final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> result = resultHandler .handle(this.rs); assertThat(result.containsKey(1)).isTrue(); assertThat(result.get(1).getFirst().getExecutor().isPresent()).isFalse(); }
@Test public void testFetchActiveFlowWithExecutor() throws Exception { initializeUnfinishedFlows(); final List<Pair<ExecutableFlow, Optional<Executor>>> activeFlowsWithExecutor = this.controller.getActiveFlowsWithExecutor(); this.unfinishedFlows.values().forEach(pair -> assertThat(activeFlowsWithExecutor .contains(new Pair<>(pair.getSecond(), pair.getFirst().getExecutor()))).isTrue()); }
/** * If a flow is already dispatched to an executor, cancel by calling Executor. Else if it's still * queued in DB, remove it from DB queue and finalize. {@inheritDoc} */ @Override public void cancelFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> unfinishedFlows = this.executorLoader .fetchUnfinishedFlows(); if (unfinishedFlows.containsKey(exFlow.getExecutionId())) { final Pair<ExecutionReference, ExecutableFlow> pair = unfinishedFlows .get(exFlow.getExecutionId()); if (pair.getFirst().getExecutor().isPresent()) { // Flow is already dispatched to an executor, so call that executor to cancel the flow. this.apiGateway .callWithReferenceByUser(pair.getFirst(), ConnectorParams.CANCEL_ACTION, userId); } else { // Flow is still queued, need to finalize it and update the status in DB. ExecutionControllerUtils.finalizeFlow(this.executorLoader, this.alerterHolder, exFlow, "Cancelled before dispatching to executor", null); } } else { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } } }
@Override public Set<String> getAllActiveExecutorServerHosts() { // Includes non primary server/hosts final HashSet<String> ports = new HashSet<>(); for (final Executor executor : this.activeExecutors) { ports.add(executor.getHost() + ":" + executor.getPort()); } // include executor which were initially active and still has flows running for (final Pair<ExecutionReference, ExecutableFlow> running : this.runningFlows .values()) { final ExecutionReference ref = running.getFirst(); if (ref.getExecutor().isPresent()) { final Executor executor = ref.getExecutor().get(); ports.add(executor.getHost() + ":" + executor.getPort()); } } return ports; }
private Map<Optional<Executor>, List<ExecutableFlow>> getFlowToExecutorMap() { final HashMap<Optional<Executor>, List<ExecutableFlow>> exFlowMap = new HashMap<>(); for (final Pair<ExecutionReference, ExecutableFlow> runningFlow : this.runningFlows .values()) { final ExecutionReference ref = runningFlow.getFirst(); final ExecutableFlow flow = runningFlow.getSecond(); final Optional<Executor> executor = ref.getExecutor(); // We can set the next check time to prevent the checking of certain // flows. if (ref.getNextCheckTime() >= System.currentTimeMillis()) { continue; } List<ExecutableFlow> flows = exFlowMap.get(executor); if (flows == null) { flows = new ArrayList<>(); exFlowMap.put(executor, flows); } flows.add(flow); } return exFlowMap; }
private void getActiveFlowsWithExecutorHelper( final List<Pair<ExecutableFlow, Optional<Executor>>> flows, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { flows.add(new Pair<>(ref.getSecond(), ref .getFirst().getExecutor())); } }
Map<String, Object> callWithReference(final ExecutionReference ref, final String action, final Pair<String, String>... params) throws ExecutorManagerException { final Executor executor = ref.getExecutor().get(); return callWithExecutionId(executor.getHost(), executor.getPort(), action, ref.getExecId(), null, params); }
Map<String, Object> callWithReferenceByUser(final ExecutionReference ref, final String action, final String user, final Pair<String, String>... params) throws ExecutorManagerException { final Executor executor = ref.getExecutor().get(); return callWithExecutionId(executor.getHost(), executor.getPort(), action, ref.getExecId(), user, params); }