private static Pair<ExecutionReference, ExecutableFlow> getPairWithExecutorInfo(final ResultSet rs, final ExecutableFlow exFlow) throws SQLException { final int executorId = rs.getInt("executorId"); final String host = rs.getString("host"); final int port = rs.getInt("port"); final Executor executor; if (host == null) { logger.warn("Executor id " + executorId + " (on execution " + exFlow.getExecutionId() + ") wasn't found"); executor = null; } else { final boolean executorStatus = rs.getBoolean("executorStatus"); executor = new Executor(executorId, host, port, executorStatus); } final ExecutionReference ref = new ExecutionReference(exFlow.getExecutionId(), executor); return new Pair<>(ref, exFlow); }
@Override public List<Pair<ExecutionReference, ExecutableFlow>> handle(final ResultSet rs) throws SQLException { if (!rs.next()) { return Collections.emptyList(); } final List<Pair<ExecutionReference, ExecutableFlow>> execFlows = new ArrayList<>(); do { final int id = rs.getInt(1); final int encodingType = rs.getInt(2); final byte[] data = rs.getBytes(3); if (data == null) { logger.error("Found a flow with empty data blob exec_id: " + id); } else { final EncodingType encType = EncodingType.fromInteger(encodingType); try { final ExecutableFlow exFlow = ExecutableFlow.createExecutableFlowFromObject( GZIPUtils.transformBytesToObject(data, encType)); final ExecutionReference ref = new ExecutionReference(id); execFlows.add(new Pair<>(ref, exFlow)); } catch (final IOException e) { throw new SQLException("Error retrieving flow data " + id, e); } } } while (rs.next()); return execFlows; } }
private void mockExecutorDoesNotExist() { this.runningExecutions.get().put(EXECUTION_ID_77, new Pair<>( new ExecutionReference(EXECUTION_ID_77, null), this.execution)); }
/** * Test running flow is finalized when its executor is removed from DB. */ @Test public void checkExecutorHealthExecutorIdRemoved() throws Exception { this.activeFlows.put(EXECUTION_ID_11, new Pair<>( new ExecutionReference(EXECUTION_ID_11, null), this.flow1)); when(this.loader.fetchExecutableFlow(EXECUTION_ID_11)).thenReturn(this.flow1); this.executorHealthChecker.checkExecutorHealth(); verify(this.loader).updateExecutableFlow(this.flow1); assertThat(this.flow1.getStatus()).isEqualTo(Status.FAILED); }
/** * Test running flow is not finalized and alert email is not sent when executor is alive. */ @Test public void checkExecutorHealthAlive() throws Exception { this.activeFlows.put(EXECUTION_ID_11, new Pair<>( new ExecutionReference(EXECUTION_ID_11, this.executor1), this.flow1)); when(this.apiGateway.callWithExecutionId(this.executor1.getHost(), this.executor1.getPort(), ConnectorParams.PING_ACTION, null, null)).thenReturn(ImmutableMap.of(ConnectorParams .STATUS_PARAM, ConnectorParams.RESPONSE_ALIVE)); this.executorHealthChecker.checkExecutorHealth(); assertThat(this.flow1.getStatus()).isEqualTo(Status.RUNNING); verifyZeroInteractions(this.alerterHolder); }
private Pair<ExecutionReference, ExecutableFlow> createExecutablePair( final String flowName, final int execId) throws IOException { final ExecutableFlow execFlow = TestUtils.createTestExecutableFlow("exectest1", flowName); execFlow.setExecutionId(execId); final ExecutionReference ref = new ExecutionReference(execId); return new Pair<>(ref, execFlow); }
@Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); this.execution = new ExecutableFlow(); this.execution.setExecutionId(EXECUTION_ID_77); this.activeExecutor = new Executor(1, "activeExecutor-1", 9999, true); this.runningExecutions = new RunningExecutions(); this.runningExecutions.get().put(EXECUTION_ID_77, new Pair<>( new ExecutionReference(EXECUTION_ID_77, this.activeExecutor), this.execution)); this.updater = new RunningExecutionsUpdater(this.updaterStage, this.alerterHolder, this.commonMetrics, this.apiGateway, this.runningExecutions, this.executionFinalizer, this.executorLoader); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
new ExecutionReference(exflow.getExecutionId());
public void checkExecutorHealthConsecutiveFailures() throws Exception { this.activeFlows.put(EXECUTION_ID_11, new Pair<>( new ExecutionReference(EXECUTION_ID_11, this.executor1), this.flow1));
@Test public void testExplicitlySpecifiedPriorities() throws IOException, InterruptedException { final ExecutableFlow flow1 = createExecutableFlow("exec1", 5, 3, 1); final ExecutableFlow flow2 = createExecutableFlow("exec2", 6, 3, 2); final ExecutableFlow flow3 = createExecutableFlow("exec3", 2, 3, 3); final ExecutionReference dummyRef = new ExecutionReference(0); final BlockingQueue<Pair<ExecutionReference, ExecutableFlow>> queue = new PriorityBlockingQueue<>(10, new ExecutableFlowPriorityComparator()); queue.put(new Pair<>(dummyRef, flow1)); queue.put(new Pair<>(dummyRef, flow2)); queue.put(new Pair<>(dummyRef, flow3)); Assert.assertEquals(flow2, queue.take().getSecond()); Assert.assertEquals(flow1, queue.take().getSecond()); Assert.assertEquals(flow3, queue.take().getSecond()); }
@Test public void testMixedSpecifiedPriorities() throws IOException, InterruptedException { final ExecutableFlow flow1 = createExecutableFlow("exec1", 3, 3, 1); final ExecutableFlow flow2 = createExecutableFlow("exec2", 2, 3, 2); final ExecutableFlow flow3 = createExecutableFlow("exec3", -2, 3, 3); final ExecutionReference dummyRef = new ExecutionReference(0); final BlockingQueue<Pair<ExecutionReference, ExecutableFlow>> queue = new PriorityBlockingQueue<>(10, new ExecutableFlowPriorityComparator()); queue.put(new Pair<>(dummyRef, flow1)); queue.put(new Pair<>(dummyRef, flow2)); queue.put(new Pair<>(dummyRef, flow3)); Assert.assertEquals(flow3, queue.take().getSecond()); Assert.assertEquals(flow1, queue.take().getSecond()); Assert.assertEquals(flow2, queue.take().getSecond()); }
private void testSetUpForRunningFlows() throws Exception { this.loader = mock(ExecutorLoader.class); this.apiGateway = mock(ExecutorApiGateway.class); this.user = TestUtils.getTestUser(); this.props.put(Constants.ConfigurationKeys.USE_MULTIPLE_EXECUTORS, "true"); //To test runningFlows, AZKABAN_QUEUEPROCESSING_ENABLED should be set to true //so that flows will be dispatched to executors. this.props.put(Constants.ConfigurationKeys.QUEUEPROCESSING_ENABLED, "true"); // allow two concurrent runs give one Flow this.props.put(Constants.ConfigurationKeys.MAX_CONCURRENT_RUNS_ONEFLOW, 2); final List<Executor> executors = new ArrayList<>(); final Executor executor1 = new Executor(1, "localhost", 12345, true); final Executor executor2 = new Executor(2, "localhost", 12346, true); executors.add(executor1); executors.add(executor2); when(this.loader.fetchActiveExecutors()).thenReturn(executors); this.manager = createExecutorManager(); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); this.flow1.setExecutionId(1); this.flow2.setExecutionId(2); final ExecutionReference ref1 = new ExecutionReference(this.flow1.getExecutionId(), executor1); final ExecutionReference ref2 = new ExecutionReference(this.flow2.getExecutionId(), executor2); this.activeFlows.put(this.flow1.getExecutionId(), new Pair<>(ref1, this.flow1)); this.activeFlows.put(this.flow2.getExecutionId(), new Pair<>(ref2, this.flow2)); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); }
@Test public void testEqualPriorities() throws IOException, InterruptedException { final ExecutableFlow flow1 = createExecutableFlow("exec1", 3, 1, 1); final ExecutableFlow flow2 = createExecutableFlow("exec2", 2, 2, 2); final ExecutableFlow flow3 = createExecutableFlow("exec3", -2, 3, 3); final ExecutableFlow flow4 = createExecutableFlow("exec3", 3, 4, 4); final ExecutionReference dummyRef = new ExecutionReference(0); final BlockingQueue<Pair<ExecutionReference, ExecutableFlow>> queue = new PriorityBlockingQueue<>(10, new ExecutableFlowPriorityComparator()); queue.put(new Pair<>(dummyRef, flow4)); queue.put(new Pair<>(dummyRef, flow1)); queue.put(new Pair<>(dummyRef, flow2)); queue.put(new Pair<>(dummyRef, flow3)); Assert.assertEquals(flow3, queue.take().getSecond()); Assert.assertEquals(flow1, queue.take().getSecond()); Assert.assertEquals(flow4, queue.take().getSecond()); Assert.assertEquals(flow2, queue.take().getSecond()); }
this.flow3.setExecutionId(3); this.flow4.setExecutionId(4); this.ref1 = new ExecutionReference(this.flow1.getExecutionId(), null); this.ref2 = new ExecutionReference(this.flow2.getExecutionId(), executor2); this.ref3 = new ExecutionReference(this.flow3.getExecutionId(), executor3);
@Test public void testEqualUpdateTimeAndPriority() throws IOException, InterruptedException { final ExecutableFlow flow1 = createExecutableFlow("exec1", 3, 1, 1); final ExecutableFlow flow2 = createExecutableFlow("exec2", 2, 2, 2); final ExecutableFlow flow3 = createExecutableFlow("exec3", -2, 2, 3); final ExecutableFlow flow4 = createExecutableFlow("exec3", 3, 4, 4); final ExecutionReference dummyRef = new ExecutionReference(0); final BlockingQueue<Pair<ExecutionReference, ExecutableFlow>> queue = new PriorityBlockingQueue<>(10, new ExecutableFlowPriorityComparator()); queue.put(new Pair<>(dummyRef, flow4)); queue.put(new Pair<>(dummyRef, flow1)); queue.put(new Pair<>(dummyRef, flow2)); queue.put(new Pair<>(dummyRef, flow3)); Assert.assertEquals(flow3, queue.take().getSecond()); Assert.assertEquals(flow1, queue.take().getSecond()); Assert.assertEquals(flow4, queue.take().getSecond()); Assert.assertEquals(flow2, queue.take().getSecond()); } }
@Override public List<Pair<ExecutionReference, ExecutableFlow>> handle(final ResultSet rs) throws SQLException { if (!rs.next()) { return Collections.emptyList(); } final List<Pair<ExecutionReference, ExecutableFlow>> execFlows = new ArrayList<>(); do { final int id = rs.getInt(1); final int encodingType = rs.getInt(2); final byte[] data = rs.getBytes(3); if (data == null) { logger.error("Found a flow with empty data blob exec_id: " + id); } else { final EncodingType encType = EncodingType.fromInteger(encodingType); try { final ExecutableFlow exFlow = ExecutableFlow.createExecutableFlowFromObject( GZIPUtils.transformBytesToObject(data, encType)); final ExecutionReference ref = new ExecutionReference(id); execFlows.add(new Pair<>(ref, exFlow)); } catch (final IOException e) { throw new SQLException("Error retrieving flow data " + id, e); } } } while (rs.next()); return execFlows; } }
executor = new Executor(executorId, host, port, executorStatus); final ExecutionReference ref = new ExecutionReference(id, executor); execFlows.put(id, new Pair<>(ref, exFlow)); } catch (final IOException e) {
ExecutionReference ref = new ExecutionReference(id, host, port); ref.setUpdateTime(updateTime);
ExecutionReference reference = new ExecutionReference(exflow.getExecutionId(), executorHost, executorPort); executorLoader.addActiveExecutableReference(reference); try {
new ExecutionReference(exflow.getExecutionId());