public static ExecutableFlow createExecutableFlowFromObject(final Object obj) { final ExecutableFlow exFlow = new ExecutableFlow(); final HashMap<String, Object> flowObj = (HashMap<String, Object>) obj; exFlow.fillExecutableFromMapObject(flowObj); return exFlow; }
public static ExecutableFlow createExecutableFlow(final Project project, final Flow flow) { final ExecutableFlow exflow = new ExecutableFlow(project, flow); exflow.addAllProxyUsers(project.getProxyUsers()); return exflow; }
private static Pair<ExecutionReference, ExecutableFlow> getExecutableFlowMetadataHelper( final ResultSet rs) throws SQLException { final Flow flow = new Flow(rs.getString("flow_id")); final Project project = new Project(rs.getInt("project_id"), null); project.setVersion(rs.getInt("version")); final ExecutableFlow exFlow = new ExecutableFlow(project, flow); exFlow.setExecutionId(rs.getInt("exec_id")); exFlow.setStatus(Status.fromInteger(rs.getInt("status"))); exFlow.setSubmitTime(rs.getLong("submit_time")); exFlow.setStartTime(rs.getLong("start_time")); exFlow.setEndTime(rs.getLong("end_time")); exFlow.setSubmitUser(rs.getString("submit_user")); return getPairWithExecutorInfo(rs, exFlow); }
return; final ExecutableFlow exFlow = new ExecutableFlow(project, flow); exFlow.setExecutionOptions(executionOptions); ret.put("submitTime", exFlow.getSubmitTime());
@Test public void testSendEmailToInvalidAddress() throws Exception { doThrow(AddressException.class).when(this.message).sendEmail(); final Flow flow = this.project.getFlow("jobe"); flow.addFailureEmails(this.receiveAddrList); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); final CommonMetrics commonMetrics = mock(CommonMetrics.class); final Emailer emailer = new Emailer(this.props, commonMetrics, this.messageCreator, this.executorLoader); emailer.alertOnError(exFlow); verify(commonMetrics, never()).markSendEmailFail(); } }
@Test public void updateExecutions() throws Exception { final ImmutableMap<String, String> map = ImmutableMap.of("test", "response"); when(this.client .httpPost(eq(new URI("http://executor-2:1234/executor")), this.params.capture())) .thenReturn(JSONUtils.toJSON(map)); final Map<String, Object> response = this.gateway .updateExecutions(new Executor(2, "executor-2", 1234, true), Collections.singletonList(new ExecutableFlow())); assertEquals(map, response); assertEquals(new Pair<>("executionId", "[-1]"), this.params.getValue().get(0)); assertEquals(new Pair<>("updatetime", "[-1]"), this.params.getValue().get(1)); assertEquals(new Pair<>("action", "update"), this.params.getValue().get(2)); assertEquals(new Pair<>("execid", "null"), this.params.getValue().get(3)); assertEquals(new Pair<>("user", null), this.params.getValue().get(4)); } }
@Test public void testCreatingExecutorSelectorWithEmptyFilterComparatorList() throws Exception { final List<Executor> executorList = new ArrayList<>(); executorList.add(new Executor(1, "host1", 80, true)); executorList.add(new Executor(2, "host2", 80, true)); executorList.add(new Executor(3, "host3", 80, true)); executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 89, 0)); executorList.get(1) .setExecutorInfo(new ExecutorInfo(50, 14095, 50, System.currentTimeMillis(), 90, 0)); executorList.get(2) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 90, 0)); final ExecutableFlow flow = new ExecutableFlow(); final ExecutorSelector selector = new ExecutorSelector(null, null); final Executor executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(2), executor); }
@Test public void uploadProject() throws ExecutorManagerException { when(this.projectLoader.getLatestProjectVersion(this.project)).thenReturn(this.VERSION); final URL resource = requireNonNull( getClass().getClassLoader().getResource("sample_flow_01.zip")); final File projectZipFile = new File(resource.getPath()); final User uploader = new User("test_user"); // to test excluding running versions in args of cleanOlderProjectVersion final ExecutableFlow runningFlow = new ExecutableFlow(this.project, new Flow("x")); runningFlow.setVersion(this.VERSION); when(this.executorLoader.fetchUnfinishedFlowsMetadata()) .thenReturn(ImmutableMap.of(-1, new Pair<>(null, runningFlow))); this.project.setVersion(this.VERSION); checkValidationReport(this.azkabanProjectLoader .uploadProject(this.project, projectZipFile, "zip", uploader, null)); verify(this.storageManager) .uploadProject(this.project, this.VERSION + 1, projectZipFile, uploader); verify(this.projectLoader).cleanOlderProjectVersion(this.project.getId(), this.VERSION - 3, Arrays.asList(this.VERSION)); }
public static ExecutableFlow createTestExecutableFlow(final String projectName, final String flowName) throws IOException { final File jsonFlowFile = ExecutionsTestUtil.getFlowFile(projectName, flowName + ".flow"); final HashMap<String, Object> flowObj = (HashMap<String, Object>) JSONUtils.parseJSONFromFile(jsonFlowFile); final Flow flow = Flow.flowFromObject(flowObj); final Project project = new Project(1, "flow"); final HashMap<String, Flow> flowMap = new HashMap<>(); flowMap.put(flow.getId(), flow); project.setFlows(flowMap); final ExecutableFlow execFlow = new ExecutableFlow(project, flow); return execFlow; }
@Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); this.execution = new ExecutableFlow(); this.execution.setExecutionId(EXECUTION_ID_77); this.activeExecutor = new Executor(1, "activeExecutor-1", 9999, true); this.runningExecutions = new RunningExecutions(); this.runningExecutions.get().put(EXECUTION_ID_77, new Pair<>( new ExecutionReference(EXECUTION_ID_77, this.activeExecutor), this.execution)); this.updater = new RunningExecutionsUpdater(this.updaterStage, this.alerterHolder, this.commonMetrics, this.apiGateway, this.runningExecutions, this.executionFinalizer, this.executorLoader); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
@Test public void testExecutorSelectorE2E() throws Exception { final List<String> filterList = new ArrayList<>(ExecutorFilter.getAvailableFilterNames()); final Map<String, Integer> comparatorMap; comparatorMap = new HashMap<>(); final List<Executor> executorList = new ArrayList<>(); executorList.add(new Executor(1, "host1", 80, true)); executorList.add(new Executor(2, "host2", 80, true)); executorList.add(new Executor(3, "host3", 80, true)); executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 89, 0)); executorList.get(1) .setExecutorInfo(new ExecutorInfo(50, 14095, 50, System.currentTimeMillis(), 90, 0)); executorList.get(2) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 90, 0)); final ExecutableFlow flow = new ExecutableFlow(); for (final String name : ExecutorComparator.getAvailableComparatorNames()) { comparatorMap.put(name, 1); } final ExecutorSelector selector = new ExecutorSelector(filterList, comparatorMap); Executor executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(0), executor); // simulate that once the flow is assigned, executor1's remaining TMP storage dropped to 2048 // now we do the getBest again executor3 is expected to be selected as it has a earlier last dispatched time. executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 4095, 50, System.currentTimeMillis(), 90, 1)); executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(2), executor); }
@Test public void testSendErrorEmail() throws Exception { final Flow flow = this.project.getFlow("jobe"); flow.addFailureEmails(this.receiveAddrList); Assert.assertNotNull(flow); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); final CommonMetrics commonMetrics = new CommonMetrics(new MetricsManager(new MetricRegistry())); final Emailer emailer = new Emailer(this.props, commonMetrics, this.messageCreator, this.executorLoader); emailer.alertOnError(exFlow); verify(this.message).addAllToAddress(this.receiveAddrList); verify(this.message).setSubject("Flow 'jobe' has failed on azkaban"); assertThat(TestUtils.readResource("errorEmail2.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
@Test public void alertOnFailedUpdate() throws Exception { final Flow flow = this.project.getFlow("jobe"); flow.addFailureEmails(this.receiveAddrList); Assert.assertNotNull(flow); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); final CommonMetrics commonMetrics = new CommonMetrics(new MetricsManager(new MetricRegistry())); final Emailer emailer = new Emailer(this.props, commonMetrics, this.messageCreator, this.executorLoader); final Executor executor = new Executor(1, "executor1-host", 1234, true); final List<ExecutableFlow> executions = Arrays.asList(exFlow, exFlow); final ExecutorManagerException exception = DefaultMailCreatorTest.createTestStracktrace(); emailer.alertOnFailedUpdate(executor, executions, exception); verify(this.message).addAllToAddress(this.receiveAddrList); verify(this.message) .setSubject("Flow status could not be updated from executor1-host on azkaban"); assertThat(TestUtils.readResource("failedUpdateMessage2.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
@Before public void setUp() throws Exception { this.defaultTz = TimeZone.getDefault(); assertNotNull(this.defaultTz); // EEST TimeZone.setDefault(TimeZone.getTimeZone("Europe/Helsinki")); DateTimeUtils.setCurrentMillisFixed(FIXED_CURRENT_TIME_MILLIS); this.mailCreator = new DefaultMailCreator(); this.executor = new Executor(1, "executor1-host", 1234, true); this.flow = new Flow("mail-creator-test"); this.project = new Project(1, "test-project"); this.options = new ExecutionOptions(); this.message = new EmailMessage("localhost", EmailMessageCreator.DEFAULT_SMTP_PORT, "", "", null); this.azkabanName = "unit-tests"; this.scheme = "http"; this.clientHostname = "localhost"; this.clientPortNumber = "8081"; final Node failedNode = new Node("test-job"); failedNode.setType("noop"); this.flow.addNode(failedNode); this.executableFlow = new ExecutableFlow(this.project, this.flow); this.executableFlow.setExecutionOptions(this.options); this.executableFlow.setStartTime(START_TIME_MILLIS); this.options.setFailureEmails(ImmutableList.of("test@example.com")); this.options.setSuccessEmails(ImmutableList.of("test@example.com")); }
@Test public void createErrorEmail() throws Exception { setJobStatus(Status.FAILED); this.executableFlow.setEndTime(END_TIME_MILLIS); this.executableFlow.setStatus(Status.FAILED); final List<ExecutableFlow> executableFlows = new ArrayList<>(); final ExecutableFlow executableFlow1 = new ExecutableFlow(this.project, this.flow); executableFlow1.setExecutionId(1); executableFlow1.setStartTime(START_TIME_MILLIS); executableFlow1.setEndTime(END_TIME_MILLIS); executableFlow1.setStatus(Status.FAILED); executableFlows.add(executableFlow1); final ExecutableFlow executableFlow2 = new ExecutableFlow(this.project, this.flow); executableFlow2.setExecutionId(2); executableFlow2.setStartTime(START_TIME_MILLIS); executableFlow2.setEndTime(END_TIME_MILLIS); executableFlow2.setStatus(Status.SUCCEEDED); executableFlows.add(executableFlow2); assertTrue(this.mailCreator.createErrorEmail( this.executableFlow, executableFlows, this.message, this.azkabanName, this.scheme, this .clientHostname, this.clientPortNumber)); assertEquals("Flow 'mail-creator-test' has failed on unit-tests", this.message.getSubject()); assertThat(TestUtils.readResource("errorEmail.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
@Test public void testExecutorFlowJson() throws Exception { final Flow flow = this.project.getFlow("jobe"); Assert.assertNotNull(flow); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); final Object obj = exFlow.toObject(); final String exFlowJSON = JSONUtils.toJSON(obj); final Map<String, Object> flowObjMap = (Map<String, Object>) JSONUtils.parseJSONFromString(exFlowJSON); final ExecutableFlow parsedExFlow = ExecutableFlow.createExecutableFlowFromObject(flowObjMap); testEquals(exFlow, parsedExFlow); }
public static ExecutableFlow createTestExecutableFlowFromYaml(final String projectName, final String flowName) throws IOException { final Project project = new Project(11, projectName); final DirectoryYamlFlowLoader loader = new DirectoryYamlFlowLoader(new Props()); loader.loadProjectFlow(project, ExecutionsTestUtil.getFlowDir(projectName)); project.setFlows(loader.getFlowMap()); project.setVersion(123); final Flow flow = project.getFlow(flowName); return new ExecutableFlow(project, flow); }
Assert.assertNotNull(flow); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); exFlow.setExecutionId(101);
private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }
Assert.assertNotNull(flow); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); Assert.assertNotNull(exFlow.getExecutableNode("joba")); Assert.assertNotNull(exFlow.getExecutableNode("jobb"));