private void addProfileMetadataUpdate(NamespacedEntityId entityId, ProfileId profileId, Map<MetadataEntity, Map<String, String>> updates) { LOG.trace("Setting profile metadata for {} to {}", entityId, profileId); updates.put(entityId.toMetadataEntity(), Collections.singletonMap(PROFILE_METADATA_KEY, profileId.getScopedName())); }
private void setProfileMetadata(NamespacedEntityId entityId, ProfileId profileId) { // if we are able to get profile from preferences or schedule properties, use it // otherwise default profile will be used metadataDataset.setProperty(entityId.toMetadataEntity(), PROFILE_METADATA_KEY, profileId.getScopedName()); }
/** * Disable the profile. After the profile is disabled, any program/schedule cannot be associated with this profile. * * @param profileId the id of the profile to disable * @throws NotFoundException if the profile is not found * @throws ProfileConflictException if the profile is already disabled * @throws MethodNotAllowedException if trying to disable the native profile */ public void disableProfile(ProfileId profileId) throws NotFoundException, ProfileConflictException, MethodNotAllowedException { if (profileId.equals(ProfileId.NATIVE)) { throw new MethodNotAllowedException(String.format("Cannot change status for Profile Native %s, " + "it should always be ENABLED", profileId.getScopedName())); } Transactionals.execute(transactional, context -> { getProfileDataset(context).disableProfile(profileId); }, NotFoundException.class, ProfileConflictException.class); }
/** * Disable the profile. After the profile is disabled, any program/schedule cannot be associated with this profile. * * @param profileId the id of the profile to disable * @throws NotFoundException if the profile is not found * @throws ProfileConflictException if the profile is already disabled * @throws MethodNotAllowedException if trying to disable the native profile */ public void disableProfile(ProfileId profileId) throws NotFoundException, ProfileConflictException, MethodNotAllowedException { if (profileId.equals(ProfileId.NATIVE)) { throw new MethodNotAllowedException(String.format("Cannot change status for Profile Native %s, " + "it should always be ENABLED", profileId.getScopedName())); } Transactionals.execute(transactional, context -> { getProfileDataset(context).disableProfile(profileId); }, NotFoundException.class, ProfileConflictException.class); }
/** * Save the profile to the profile store. By default the profile status will be enabled. * * @param profileId the id of the profile to save * @param profile the information of the profile * @throws MethodNotAllowedException if trying to update the Native profile */ public void saveProfile(ProfileId profileId, Profile profile) throws MethodNotAllowedException { Transactionals.execute(transactional, context -> { ProfileDataset dataset = getProfileDataset(context); if (profileId.equals(ProfileId.NATIVE)) { try { dataset.getProfile(profileId); throw new MethodNotAllowedException(String.format("Profile Native %s already exists. It cannot be updated", profileId.getScopedName())); } catch (NotFoundException e) { // if native profile is not found, we can add it to the dataset } } dataset.saveProfile(profileId, profile); }, MethodNotAllowedException.class); }
/** * Save the profile to the profile store. By default the profile status will be enabled. * * @param profileId the id of the profile to save * @param profile the information of the profile * @throws MethodNotAllowedException if trying to update the Native profile */ public void saveProfile(ProfileId profileId, Profile profile) throws MethodNotAllowedException { Transactionals.execute(transactional, context -> { ProfileDataset dataset = getProfileDataset(context); if (profileId.equals(ProfileId.NATIVE)) { try { dataset.getProfile(profileId); throw new MethodNotAllowedException(String.format("Profile Native %s already exists. It cannot be updated", profileId.getScopedName())); } catch (NotFoundException e) { // if native profile is not found, we can add it to the dataset } } dataset.saveProfile(profileId, profile); }, MethodNotAllowedException.class); }
private void setStart(ProgramRunId id, Map<String, String> runtimeArgs, Map<String, String> systemArgs, ArtifactId artifactId) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } store.setProvisioning(id, runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(id, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); }
/** * Deletes the profile from the profile store. Native profile cannot be deleted. * Other profile deletion must satisfy the following: * 1. Profile must exist and must be DISABLED * 2. Profile must not be assigned to any entities. Profiles can be assigned to an entity by setting a preference * or a schedule property. * 3. There must be no active program runs using this profile * * @param profileId the id of the profile to delete * @throws NotFoundException if the profile is not found * @throws ProfileConflictException if the profile is enabled * @throws MethodNotAllowedException if trying to delete the Native profile */ public void deleteProfile(ProfileId profileId) throws MethodNotAllowedException, NotFoundException, ProfileConflictException { if (profileId.equals(ProfileId.NATIVE)) { throw new MethodNotAllowedException(String.format("Profile Native %s cannot be deleted.", profileId.getScopedName())); } Transactionals.execute(transactional, context -> { ProfileDataset profileDataset = getProfileDataset(context); Profile profile = profileDataset.getProfile(profileId); AppMetadataStore appMetadataStore = AppMetadataStore.create(cConf, context, datasetFramework); deleteProfile(profileDataset, appMetadataStore, profileId, profile); }, NotFoundException.class, ProfileConflictException.class); deleteMetrics(profileId); }
/** * Deletes the profile from the profile store. Native profile cannot be deleted. * Other profile deletion must satisfy the following: * 1. Profile must exist and must be DISABLED * 2. Profile must not be assigned to any entities. Profiles can be assigned to an entity by setting a preference * or a schedule property. * 3. There must be no active program runs using this profile * * @param profileId the id of the profile to delete * @throws NotFoundException if the profile is not found * @throws ProfileConflictException if the profile is enabled * @throws MethodNotAllowedException if trying to delete the Native profile */ public void deleteProfile(ProfileId profileId) throws MethodNotAllowedException, NotFoundException, ProfileConflictException { if (profileId.equals(ProfileId.NATIVE)) { throw new MethodNotAllowedException(String.format("Profile Native %s cannot be deleted.", profileId.getScopedName())); } Transactionals.execute(transactional, context -> { ProfileDataset profileDataset = getProfileDataset(context); Profile profile = profileDataset.getProfile(profileId); AppMetadataStore appMetadataStore = AppMetadataStore.create(cConf, context, datasetFramework); deleteProfile(profileDataset, appMetadataStore, profileId, profile); }, NotFoundException.class, ProfileConflictException.class); deleteMetrics(profileId); }
@Test public void testGetProfileId() { // should get null profile id if the args is empty Assert.assertFalse(SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, Collections.emptyMap()).isPresent()); Map<String, String> args = new HashMap<>(); args.put("system.log.level", "DEBUG"); args.put("system.log.leveldummyKey", "ERROR"); // Having other unrelated args should also get null profile id Assert.assertFalse(SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).isPresent()); // without scope the profile will be considered in user scope ProfileId expected = NamespaceId.DEFAULT.profile("MyProfile"); args.put("system.profile.name", expected.getProfile()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); // put a profile with scope SYSTEM, the profile we get should be in system namespace expected = NamespaceId.SYSTEM.profile("MyProfile"); args.put("system.profile.name", expected.getScopedName()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); // put a profile with scope USER, the profile we get should be in the user namespace expected = NamespaceId.DEFAULT.profile("MyProfile"); args.put("system.profile.name", expected.getScopedName()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); } }
private void setStartAndRunning(Store store, ProgramRunId id, Map<String, String> runtimeArgs, Map<String, String> systemArgs) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } long startTime = RunIds.getTime(id.getRun(), TimeUnit.SECONDS); store.setProvisioning(id, runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), ARTIFACT_ID); store.setProvisioned(id, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(id, startTime + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); }
private void setStartAndRunning(Store store, ProgramRunId id, Map<String, String> runtimeArgs, Map<String, String> systemArgs) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } long startTime = RunIds.getTime(id.getRun(), TimeUnit.SECONDS); store.setProvisioning(id, runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), ARTIFACT_ID); store.setProvisioned(id, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(id, startTime + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); }
private void setStartAndRunning(Store store, ProgramId id, String pid, Map<String, String> runtimeArgs, Map<String, String> systemArgs, ArtifactId artifactId) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } long startTime = RunIds.getTime(pid, TimeUnit.SECONDS); store.setProvisioning(id.run(pid), runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(id.run(pid), 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id.run(pid), null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(id.run(pid), startTime + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); }
/** Adds runs which have workflows associated with them * * @param store store instance * @param workflowName name of the workflow * @param workflowRunId run ID associated with all program runs * @param runs list ofo runs to be added */ private void addWorkflowRuns(Store store, String workflowName, String workflowRunId, ProgramRunId... runs) { Map<String, String> workflowIDMap = new HashMap<>(); Map<String, String> emptyMap = ImmutableMap.of(); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NAME, workflowName); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NODE_ID, "workflowNodeId"); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId); workflowIDMap.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()); for (ProgramRunId run : runs) { ArtifactId artifactId = run.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); store.setProvisioning(run, emptyMap, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(run, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(run, null, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(run, RunIds.getTime(run.getRun(), TimeUnit.SECONDS) + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); } }
@Test public void testAppSpecNotRequiredToWriteState() throws Exception { ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s"); Map<String, String> systemArguments = new HashMap<>(); systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()); systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()); ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments()); ProgramRunId runId = programId.run(RunIds.generate()); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec); programStateWriter.start(runId, programOptions, null, programDescriptor); Tasks.waitFor(ProgramRunStatus.STARTING, () -> txnl.execute(() -> { RunRecordMeta meta = metadataStoreDataset.getRun(runId); if (meta == null) { return null; } Assert.assertEquals(artifactId, meta.getArtifactId()); return meta.getStatus(); }), 10, TimeUnit.SECONDS); programStateWriter.completed(runId); }
Collections.singletonMap("profile", ProfileId.NATIVE.getScopedName())); Assert.assertEquals(ProfileId.NATIVE.getScopedName(), mds.getProperties(workflowId.toMetadataEntity()).get("profile"));
@Test public void testAddDeleteScheduleWithProfileProperty() throws Exception { // put my profile and by default it is enabled ProfileId profileId = NS_ID.profile("MyProfile"); putProfile(profileId, Profile.NATIVE, 200); // add a schedule, it should succeed since the profile is enabled. ProgramSchedule tsched1 = new ProgramSchedule("tsched1", "one time schedule", PROG1_ID, ImmutableMap.of("prop1", "nn", SystemArguments.PROFILE_NAME, profileId.getScopedName()), new TimeTrigger("* * ? * 1"), ImmutableList.<Constraint>of()); scheduler.addSchedule(tsched1); // now disable the profile and delete, deletion should fail because the profile is now associated with the schedule disableProfile(profileId, 200); deleteProfile(profileId, 409); // delete it scheduler.deleteSchedule(TSCHED1_ID); // now deletion should succeed since it should remove assignment from the profile deleteProfile(profileId, 200); }
@Test public void testStartProgramWithDisabledProfile() throws Exception { // put my profile and disable it, using this profile to start program should fail ProfileId profileId = new NamespaceId(TEST_NAMESPACE1).profile("MyProfile"); Profile profile = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner()); putProfile(profileId, profile, 200); disableProfile(profileId, 200); // deploy, check the status deploy(AppWithWorkflow.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); ProgramId programId = new NamespaceId(TEST_NAMESPACE1).app(AppWithWorkflow.NAME).workflow(AppWithWorkflow.SampleWorkflow.NAME); // workflow is stopped initially Assert.assertEquals(STOPPED, getProgramStatus(programId)); // start workflow should give a 409 since we have a runtime argument associated with a disabled profile startProgram(programId, Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()), 409); Assert.assertEquals(STOPPED, getProgramStatus(programId)); // use native profile to start workflow should work since it is always enabled. // the workflow should start but fail because we are not passing in required runtime args. int runs = getProgramRuns(programId, ProgramRunStatus.FAILED).size(); startProgram(programId, Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()), 200); // wait for the workflow to stop and check the status Tasks.waitFor(runs + 1, () -> getProgramRuns(programId, ProgramRunStatus.FAILED).size(), 60, TimeUnit.SECONDS); }
@Test public void testGetProgramProfile() { ProfileId profileId = NamespaceId.DEFAULT.profile("p"); Map<String, String> args = Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()); ApplicationId appId = NamespaceId.DEFAULT.app("a"); ProgramId mrId = appId.mr("mr"); ProgramId serviceId = appId.service("serv"); ProgramId sparkId = appId.spark("spark"); ProgramId workerId = appId.worker("worker"); ProgramId workflowID = appId.workflow("wf"); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(mrId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(serviceId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(sparkId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(workerId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(workflowID, args)); }
@Test public void testProfileInRunRecord() throws Exception { AppMetadataStore store = getMetadataStore("testProfileInRunRecord"); TransactionExecutor txnl = getTxExecutor(store); ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate()); ProfileId profileId = NamespaceId.DEFAULT.profile("MyProfile"); txnl.execute(() -> { long startSourceId = 1L; store.recordProgramProvisioning(runId, null, Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()), AppFabricTestHelper.createSourceId(startSourceId), ARTIFACT_ID); // the profile id should be there after the provisioning stage RunRecordMeta run = store.getRun(runId); Assert.assertNotNull(run); Assert.assertEquals(profileId, run.getProfileId()); store.recordProgramProvisioned(runId, 0, AppFabricTestHelper.createSourceId(startSourceId + 1)); store.recordProgramStart(runId, null, ImmutableMap.of(), AppFabricTestHelper.createSourceId(startSourceId + 2)); store.recordProgramRunning(runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS), null, AppFabricTestHelper.createSourceId(startSourceId + 3)); store.recordProgramStop(runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS), ProgramRunStatus.KILLED, null, AppFabricTestHelper.createSourceId(startSourceId + 4)); run = store.getRun(runId); Assert.assertNotNull(run); Assert.assertEquals(profileId, run.getProfileId()); }); }