private ProfileId getId() { return NamespaceId.SYSTEM.profile(name); }
private ProfileId getId() { return NamespaceId.SYSTEM.profile(name); }
private ProfileId getValidatedProfile(NamespaceId namespaceId, String profileName) throws BadRequestException { try { return namespaceId.profile(profileName); } catch (IllegalArgumentException e) { throw new BadRequestException(e.getMessage(), e); } }
private ProfileId getValidatedProfile(NamespaceId namespaceId, String profileName) throws BadRequestException { try { return namespaceId.profile(profileName); } catch (IllegalArgumentException e) { throw new BadRequestException(e.getMessage(), e); } }
/** * Get the profile id for the given scoped profile name. A scoped name starts with an entity scope, a ':', * then the profile name. If there is no ':', it is assumed that it is user scope. * * @param namespaceId the namespace to use if user scoped * @param scopedName the scoped name * @return the profile id * @throws IllegalArgumentException if the scope is invalid or the profile name is invalid. */ public static ProfileId fromScopedName(NamespaceId namespaceId, String scopedName) { int idx = scopedName.indexOf(':'); if (idx < 0) { return namespaceId.profile(scopedName); } if (idx == scopedName.length()) { throw new IllegalArgumentException("Invalid scoped profile " + scopedName + ". Cannot end with a ':'."); } EntityScope scope = EntityScope.valueOf(scopedName.substring(0, idx).toUpperCase()); String profileName = scopedName.substring(idx + 1); return scope == EntityScope.SYSTEM ? NamespaceId.SYSTEM.profile(profileName) : namespaceId.profile(profileName); }
/** * Get the profile id for the given scoped profile name. A scoped name starts with an entity scope, a ':', * then the profile name. If there is no ':', it is assumed that it is user scope. * * @param namespaceId the namespace to use if user scoped * @param scopedName the scoped name * @return the profile id * @throws IllegalArgumentException if the scope is invalid or the profile name is invalid. */ public static ProfileId fromScopedName(NamespaceId namespaceId, String scopedName) { int idx = scopedName.indexOf(':'); if (idx < 0) { return namespaceId.profile(scopedName); } if (idx == scopedName.length()) { throw new IllegalArgumentException("Invalid scoped profile " + scopedName + ". Cannot end with a ':'."); } EntityScope scope = EntityScope.valueOf(scopedName.substring(0, idx).toUpperCase()); String profileName = scopedName.substring(idx + 1); return scope == EntityScope.SYSTEM ? NamespaceId.SYSTEM.profile(profileName) : namespaceId.profile(profileName); }
@Test public void testSystemNamespaceProfilesNotAllowed() throws Exception { listProfiles(NamespaceId.SYSTEM, false, 405); getProfile(NamespaceId.SYSTEM.profile("abc"), 405); disableProfile(NamespaceId.SYSTEM.profile("abc"), 405); enableProfile(NamespaceId.SYSTEM.profile("abc"), 405); Profile profile = new Profile("abc", "label", "desc", new ProvisionerInfo("xyz", Collections.emptyList())); putProfile(NamespaceId.SYSTEM.profile("abc"), profile, 405); }
/** * Delete all profiles in a given namespace. Deleting all profiles in SYSTEM namespace is not allowed. * * @param namespaceId the id of the namespace */ public void deleteAllProfiles(NamespaceId namespaceId) throws MethodNotAllowedException, NotFoundException, ProfileConflictException { if (namespaceId.equals(NamespaceId.SYSTEM)) { throw new MethodNotAllowedException("Deleting all system profiles is not allowed."); } List<ProfileId> deleted = new ArrayList<>(); Transactionals.execute(transactional, context -> { ProfileDataset profileDataset = getProfileDataset(context); AppMetadataStore appMetadataStore = AppMetadataStore.create(cConf, context, datasetFramework); List<Profile> profiles = profileDataset.getProfiles(namespaceId, false); for (Profile profile : profiles) { ProfileId profileId = namespaceId.profile(profile.getName()); deleteProfile(profileDataset, appMetadataStore, profileId, profile); deleted.add(profileId); } }, ProfileConflictException.class, NotFoundException.class); // delete the metrics for (ProfileId profileId : deleted) { deleteMetrics(profileId); } }
/** * Delete all profiles in a given namespace. Deleting all profiles in SYSTEM namespace is not allowed. * * @param namespaceId the id of the namespace */ public void deleteAllProfiles(NamespaceId namespaceId) throws MethodNotAllowedException, NotFoundException, ProfileConflictException { if (namespaceId.equals(NamespaceId.SYSTEM)) { throw new MethodNotAllowedException("Deleting all system profiles is not allowed."); } List<ProfileId> deleted = new ArrayList<>(); Transactionals.execute(transactional, context -> { ProfileDataset profileDataset = getProfileDataset(context); AppMetadataStore appMetadataStore = AppMetadataStore.create(cConf, context, datasetFramework); List<Profile> profiles = profileDataset.getProfiles(namespaceId, false); for (Profile profile : profiles) { ProfileId profileId = namespaceId.profile(profile.getName()); deleteProfile(profileDataset, appMetadataStore, profileId, profile); deleted.add(profileId); } }, ProfileConflictException.class, NotFoundException.class); // delete the metrics for (ProfileId profileId : deleted) { deleteMetrics(profileId); } }
@Test public void testGetProfileId() { // should get null profile id if the args is empty Assert.assertFalse(SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, Collections.emptyMap()).isPresent()); Map<String, String> args = new HashMap<>(); args.put("system.log.level", "DEBUG"); args.put("system.log.leveldummyKey", "ERROR"); // Having other unrelated args should also get null profile id Assert.assertFalse(SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).isPresent()); // without scope the profile will be considered in user scope ProfileId expected = NamespaceId.DEFAULT.profile("MyProfile"); args.put("system.profile.name", expected.getProfile()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); // put a profile with scope SYSTEM, the profile we get should be in system namespace expected = NamespaceId.SYSTEM.profile("MyProfile"); args.put("system.profile.name", expected.getScopedName()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); // put a profile with scope USER, the profile we get should be in the user namespace expected = NamespaceId.DEFAULT.profile("MyProfile"); args.put("system.profile.name", expected.getScopedName()); Assert.assertEquals(expected, SystemArguments.getProfileIdFromArgs(NamespaceId.DEFAULT, args).get()); } }
@Test public void testListAndGetProfiles() throws Exception { // no profile should be there in default namespace List<Profile> profiles = listProfiles(NamespaceId.DEFAULT, false, 200); Assert.assertEquals(Collections.emptyList(), profiles); // try to list all profiles including system namespace before putting a new one, there should only exist a default // profile profiles = listProfiles(NamespaceId.DEFAULT, true, 200); Assert.assertEquals(Collections.singletonList(Profile.NATIVE), profiles); // test get single profile endpoint ProfileId profileId = NamespaceId.DEFAULT.profile("p1"); Profile expected = new Profile("p1", "label", "my profile for testing", new ProvisionerInfo(MockProvisioner.NAME, PROPERTY_SUMMARIES)); putProfile(profileId, expected, 200); Profile actual = getProfile(profileId, 200).get(); Assert.assertEquals(expected, actual); // get a nonexisting profile should get a not found code getProfile(NamespaceId.DEFAULT.profile("nonExisting"), 404); }
ProfileId profileId = NamespaceId.DEFAULT.profile("p"); profileService.saveProfile(profileId, profile);
@Test public void testProfileCreationTime() throws Exception { ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile"); long creationTime = TimeUnit.SECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS); Profile profile = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), ProfileStatus.ENABLED, Profile.NATIVE.getProvisioner(), creationTime); profileService.saveProfile(myProfile, profile); Assert.assertEquals(creationTime, profileService.getProfile(myProfile).getCreatedTsSeconds()); }
@Test public void testExistingIsUnmodified() throws Exception { // write a profile ProfileId profileId = NamespaceId.SYSTEM.profile("p1"); List<ProvisionerPropertyValue> properties = new ArrayList<>(); properties.add(new ProvisionerPropertyValue("name1", "val1", true)); properties.add(new ProvisionerPropertyValue("name2", "val2", true)); ProvisionerInfo provisionerInfo = new ProvisionerInfo(MockProvisioner.NAME, properties); Profile profile = new Profile(profileId.getProfile(), "profile label", "profile description", EntityScope.SYSTEM, provisionerInfo); profileService.saveProfile(profileId, profile); // run the bootstrap step and make sure it succeeded SystemProfileCreator.Arguments arguments = new SystemProfileCreator.Arguments(profile.getName(), "different label", "different desciption", profile.getProvisioner()); BootstrapStepResult result = profileCreator.execute("label", GSON.toJsonTree(arguments).getAsJsonObject()); BootstrapStepResult expected = new BootstrapStepResult("label", BootstrapStepResult.Status.SUCCEEDED); Assert.assertEquals(expected, result); // check that it didn't overwrite the existing profile Assert.assertEquals(profile, profileService.getProfile(profileId)); } }
@Test public void testAddDeleteScheduleWithProfileProperty() throws Exception { // put my profile and by default it is enabled ProfileId profileId = NS_ID.profile("MyProfile"); putProfile(profileId, Profile.NATIVE, 200); // add a schedule, it should succeed since the profile is enabled. ProgramSchedule tsched1 = new ProgramSchedule("tsched1", "one time schedule", PROG1_ID, ImmutableMap.of("prop1", "nn", SystemArguments.PROFILE_NAME, profileId.getScopedName()), new TimeTrigger("* * ? * 1"), ImmutableList.<Constraint>of()); scheduler.addSchedule(tsched1); // now disable the profile and delete, deletion should fail because the profile is now associated with the schedule disableProfile(profileId, 200); deleteProfile(profileId, 409); // delete it scheduler.deleteSchedule(TSCHED1_ID); // now deletion should succeed since it should remove assignment from the profile deleteProfile(profileId, 200); }
@Test public void testCreation() throws Exception { ProfileId profileId = NamespaceId.SYSTEM.profile("p1"); try { profileService.getProfile(profileId); Assert.fail("profile should not exist."); } catch (NotFoundException e) { // expected } List<ProvisionerPropertyValue> properties = new ArrayList<>(); properties.add(new ProvisionerPropertyValue("name1", "val1", true)); properties.add(new ProvisionerPropertyValue("name2", "val2", true)); ProvisionerInfo provisionerInfo = new ProvisionerInfo(MockProvisioner.NAME, properties); Profile profile = new Profile(profileId.getProfile(), "profile label", "profile description", EntityScope.SYSTEM, provisionerInfo); SystemProfileCreator.Arguments arguments = new SystemProfileCreator.Arguments(profile.getName(), profile.getLabel(), profile.getDescription(), profile.getProvisioner()); BootstrapStepResult result = profileCreator.execute("label", GSON.toJsonTree(arguments).getAsJsonObject()); BootstrapStepResult expected = new BootstrapStepResult("label", BootstrapStepResult.Status.SUCCEEDED); Assert.assertEquals(expected, result); Assert.assertEquals(profile, profileService.getProfile(profileId)); }
@Test public void testProfileMetrics() throws Exception { ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate()); ProfileId profileId = NamespaceId.DEFAULT.profile("myProfile"); MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class); MetricStore metricStore = injector.getInstance(MetricStore.class); // There are 5 nodes, we emit the metrics each 2 mins, so each time the node minute should go up by 10 min ProfileMetricService scheduledService = new ProfileMetricService(collectionService, runId, profileId, 5, 2, null); // emit and verify the results scheduledService.emitMetric(); Tasks.waitFor(10L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); scheduledService.emitMetric(); Tasks.waitFor(20L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); scheduledService.emitMetric(); Tasks.waitFor(30L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); }
@Test public void testStartProgramWithDisabledProfile() throws Exception { // put my profile and disable it, using this profile to start program should fail ProfileId profileId = new NamespaceId(TEST_NAMESPACE1).profile("MyProfile"); Profile profile = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner()); putProfile(profileId, profile, 200); disableProfile(profileId, 200); // deploy, check the status deploy(AppWithWorkflow.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); ProgramId programId = new NamespaceId(TEST_NAMESPACE1).app(AppWithWorkflow.NAME).workflow(AppWithWorkflow.SampleWorkflow.NAME); // workflow is stopped initially Assert.assertEquals(STOPPED, getProgramStatus(programId)); // start workflow should give a 409 since we have a runtime argument associated with a disabled profile startProgram(programId, Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()), 409); Assert.assertEquals(STOPPED, getProgramStatus(programId)); // use native profile to start workflow should work since it is always enabled. // the workflow should start but fail because we are not passing in required runtime args. int runs = getProgramRuns(programId, ProgramRunStatus.FAILED).size(); startProgram(programId, Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()), 200); // wait for the workflow to stop and check the status Tasks.waitFor(runs + 1, () -> getProgramRuns(programId, ProgramRunStatus.FAILED).size(), 60, TimeUnit.SECONDS); }
@Test public void testGetProgramProfile() { ProfileId profileId = NamespaceId.DEFAULT.profile("p"); Map<String, String> args = Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()); ApplicationId appId = NamespaceId.DEFAULT.app("a"); ProgramId mrId = appId.mr("mr"); ProgramId serviceId = appId.service("serv"); ProgramId sparkId = appId.spark("spark"); ProgramId workerId = appId.worker("worker"); ProgramId workflowID = appId.workflow("wf"); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(mrId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(serviceId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(sparkId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(workerId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(workflowID, args)); }
@Test public void testProfileMetricsDeletion() throws Exception { ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile"); Profile profile = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner()); ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate()); // create and disable the profile profileService.saveProfile(myProfile, profile); profileService.disableProfile(myProfile); // emit some metrics MetricsCollectionService metricService = injector.getInstance(MetricsCollectionService.class); MetricsContext metricsContext = metricService.getContext(getMetricsTags(runId, myProfile)); metricsContext.increment(Constants.Metrics.Program.PROGRAM_NODE_MINUTES, 30L); MetricStore metricStore = injector.getInstance(MetricStore.class); Tasks.waitFor(30L, () -> getMetric(metricStore, runId, myProfile, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); // delete and verify the metrics are gone profileService.deleteProfile(myProfile); Tasks.waitFor(0L, () -> getMetric(metricStore, runId, myProfile, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); }