private void writeProgramSystemMetadata(ApplicationId appId, ProgramType programType, Iterable<? extends ProgramSpecification> specs) { for (ProgramSpecification spec : specs) { ProgramId programId = appId.program(programType, spec.getName()); Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, programId.toMetadataEntity()); ProgramSystemMetadataWriter writer = new ProgramSystemMetadataWriter(metadataStore, programId, spec, !properties.isEmpty()); writer.write(); } } }
/** * Delete the metadata for the application and the programs. */ private void deleteAppMetadata(ApplicationId appId, ApplicationSpecification appSpec) { // Remove metadata for the Application itself. metadataStore.removeMetadata(appId.toMetadataEntity()); // Remove metadata for the programs of the Application // TODO: Need to remove this we support prefix search of metadata type. // See https://issues.cask.co/browse/CDAP-3669 for (ProgramId programId : getAllPrograms(appId, appSpec)) { metadataStore.removeMetadata(programId.toMetadataEntity()); } }
/** * Remove the profile metadata according to the message, currently only meant for application and schedule. */ private void removeProfileMetadata(MetadataMessage message) { EntityId entity = message.getEntityId(); Map<MetadataEntity, Set<String>> toRemove = new HashMap<>(); // We only care about application and schedules. if (entity.getEntityType().equals(EntityType.APPLICATION)) { ApplicationId appId = (ApplicationId) message.getEntityId(); ApplicationSpecification appSpec = message.getPayload(GSON, ApplicationSpecification.class); for (ProgramId programId : getAllProfileAllowedPrograms(appSpec, appId)) { toRemove.put(programId.toMetadataEntity(), PROFILE_METADATA_KEY_SET); } for (ScheduleId scheduleId : getSchedulesInApp(appId, appSpec.getProgramSchedules())) { toRemove.put(scheduleId.toMetadataEntity(), PROFILE_METADATA_KEY_SET); } } else if (entity.getEntityType().equals(EntityType.SCHEDULE)) { toRemove.put(entity.toMetadataEntity(), PROFILE_METADATA_KEY_SET); } if (!toRemove.isEmpty()) { metadataStore.removeProperties(MetadataScope.SYSTEM, toRemove); } }
/** * Delete the metadata for the application and the programs. */ private void deleteAppMetadata(ApplicationId appId, ApplicationSpecification appSpec) { // Remove metadata for the Application itself. metadataPublisher.publish(NamespaceId.SYSTEM, new MetadataOperation.Drop(appId.toMetadataEntity())); // Remove metadata for the programs of the Application // TODO: Need to remove this we support prefix search of metadata type. // See https://issues.cask.co/browse/CDAP-3669 for (ProgramId programId : getAllPrograms(appId, appSpec)) { metadataPublisher.publish(NamespaceId.SYSTEM, new MetadataOperation.Drop(programId.toMetadataEntity())); } }
@Override public void process(ApplicationDeployable appSpec) throws Exception { List<ProgramSpecification> deletedSpecs = store.getDeletedProgramSpecifications(appSpec.getApplicationId(), appSpec.getSpecification()); // TODO: this should also delete logs and run records (or not?), and do it for all program types [CDAP-2187] Set<ProgramId> deletedPrograms = new HashSet<>(); for (ProgramSpecification spec : deletedSpecs) { //call the deleted spec ProgramType type = ProgramTypes.fromSpecification(spec); ProgramId programId = appSpec.getApplicationId().program(type, spec.getName()); programTerminator.stop(programId); programScheduler.deleteSchedules(programId); programScheduler.modifySchedulesTriggeredByDeletedProgram(programId); // Remove metadata for the deleted program metadataPublisher.publish(NamespaceId.SYSTEM, new MetadataOperation.Drop(programId.toMetadataEntity())); deletedPrograms.add(programId); } deleteMetrics(deletedPrograms); emit(appSpec); }
/** * Remove the profile metadata according to the message, currently only meant for application and schedule. */ private void removeProfileMetadata(MetadataMessage message) { EntityId entity = message.getEntityId(); // We only care about application and schedules. if (entity.getEntityType().equals(EntityType.APPLICATION)) { ApplicationId appId = (ApplicationId) message.getEntityId(); ApplicationSpecification appSpec = message.getPayload(GSON, ApplicationSpecification.class); for (ProgramId programId : getProgramsWithType(appId, ProgramType.WORKFLOW, appSpec.getWorkflows())) { metadataDataset.removeProperties(programId.toMetadataEntity(), Collections.singleton(PROFILE_METADATA_KEY)); } for (ScheduleId scheduleId : getSchedulesInApp(appId, appSpec.getProgramSchedules())) { metadataDataset.removeProperties(scheduleId.toMetadataEntity(), Collections.singleton(PROFILE_METADATA_KEY)); } } if (entity.getEntityType().equals(EntityType.SCHEDULE)) { metadataDataset.removeProperties(message.getEntityId().toMetadataEntity(), Collections.singleton(PROFILE_METADATA_KEY)); } }
mds.addProperties(MetadataScope.SYSTEM, workflowId.toMetadataEntity(), Collections.singletonMap("profile", ProfileId.NATIVE.getScopedName())); Assert.assertEquals(ProfileId.NATIVE.getScopedName(), mds.getProperties(workflowId.toMetadataEntity()).get("profile")); Tasks.waitFor(Collections.emptyMap(), () -> mds.getProperties(workflowId.toMetadataEntity()), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
@Override public void process(ApplicationDeployable appSpec) throws Exception { List<ProgramSpecification> deletedSpecs = store.getDeletedProgramSpecifications(appSpec.getApplicationId(), appSpec.getSpecification()); // TODO: this should also delete logs and run records (or not?), and do it for all program types [CDAP-2187] List<String> deletedFlows = Lists.newArrayList(); for (ProgramSpecification spec : deletedSpecs) { //call the deleted spec ProgramType type = ProgramTypes.fromSpecification(spec); final ProgramId programId = appSpec.getApplicationId().program(type, spec.getName()); programTerminator.stop(programId); programScheduler.deleteSchedules(programId); programScheduler.modifySchedulesTriggeredByDeletedProgram(programId); // drop all queues and stream states of a deleted flow if (ProgramType.FLOW.equals(type)) { FlowUtils.clearDeletedFlow(impersonator, queueAdmin, streamConsumerFactory, programId, (FlowSpecification) spec); deletedFlows.add(programId.getEntityName()); } // Remove metadata for the deleted program metadataStore.removeMetadata(programId.toMetadataEntity()); } if (!deletedFlows.isEmpty()) { deleteMetrics(appSpec.getApplicationId(), deletedFlows); } emit(appSpec); }
private void generateMetadataUpdates() { store.addTags(MetadataScope.USER, dataset.toMetadataEntity(), datasetTags); store.addProperties(MetadataScope.USER, app.toMetadataEntity(), appProperties); store.addTags(MetadataScope.USER, app.toMetadataEntity(), appTags); store.addTags(MetadataScope.USER, service.toMetadataEntity(), tags); store.removeTags(MetadataScope.USER, service.toMetadataEntity()); store.removeTags(MetadataScope.USER, dataset.toMetadataEntity(), datasetTags); store.removeMetadata(MetadataScope.USER, app.toMetadataEntity()); }
@Test public void testGetTargetTypeChild() { ApplicationId expectedAppId = new ApplicationId("ns1", "app1"); MDSKey mdsValueKey = MetadataKey.createValueRowKey(expectedAppId.toMetadataEntity(), "key1"); ProgramId expectedProgramId = expectedAppId.spark("spark1"); MDSKey mdsValueKey2 = MetadataKey.createValueRowKey(expectedProgramId.toMetadataEntity(), "key2"); // assert that the key for parent child are independent and correct MetadataEntity actualAppId = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey()); Assert.assertEquals(expectedAppId.toMetadataEntity(), actualAppId); MetadataEntity actualProgramId = MetadataKey.extractMetadataEntityFromKey(mdsValueKey2.getKey()); Assert.assertEquals(expectedProgramId.toMetadataEntity(), actualProgramId); }
Assert.assertEquals(Collections.emptyMap(), mds.getProperties(workflowId.toMetadataEntity())); Tasks.waitFor(myProfile.getScopedName(), () -> mds.getProperties(workflowId.toMetadataEntity()).get("profile"), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); () -> mds.getProperties(workflowId.toMetadataEntity()).get("profile"), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); } finally { profileService.disableProfile(myProfile); profileService.deleteProfile(myProfile); mds.removeMetadata(workflowId.toMetadataEntity());
Set<String> streamUserTags = ImmutableSet.of("tag3", "tag4"); Set<String> sysTags = ImmutableSet.of("sysTag1"); store.addProperties(MetadataScope.USER, service1.toMetadataEntity(), userProps); store.addProperties(MetadataScope.SYSTEM, service1.toMetadataEntity(), systemProps); store.addTags(MetadataScope.USER, service1.toMetadataEntity(), userTags); store.addTags(MetadataScope.SYSTEM, service1.toMetadataEntity(), sysTags); store.addTags(MetadataScope.USER, dataset2.toMetadataEntity(), streamUserTags); store.removeTags(MetadataScope.USER, dataset2.toMetadataEntity(), streamUserTags);
ProgramId programId2 = new ApplicationId("ns", "app", "2").program(ProgramType.SERVICE, "s"); // custom version mdsValueKey = MetadataKey.createValueRowKey(programId1.toMetadataEntity(), "key1"); actual = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey()); Assert.assertEquals(programId1.toMetadataEntity(), actual); mdsValueKey = MetadataKey.createValueRowKey(programId2.toMetadataEntity(), "key1"); actual = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey()); Assert.assertEquals(programId1.toMetadataEntity(), actual);
@Test public void testMesssages() { // test dataset InvalidMetadataException invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity(), "error"); String expectedMessage = "Unable to set metadata for dataset: ds " + "which exists in namespace: default. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); // test program invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.app("app").program(ProgramType.WORKER, "wk") .toMetadataEntity(), "error"); expectedMessage = "Unable to set metadata for worker: wk in application: app of version: -SNAPSHOT deployed in " + "namespace: default. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); // test custom entity MetadataEntity customEntity = MetadataEntity.builder(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity()) .appendAsType("field", "empName").build(); invalidMetadataException = new InvalidMetadataException(customEntity, "error"); expectedMessage = "Unable to set metadata for namespace=default,dataset=ds,field=empName of type 'field'. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); } }
final MetadataEntity app1 = app1Id.toMetadataEntity(); final MetadataEntity app2 = app2Id.toMetadataEntity(); final MetadataEntity program1 = app1Id.worker("wk1").toMetadataEntity(); final MetadataEntity dataset1 = ns1Id.dataset("ds1").toMetadataEntity(); final MetadataEntity dataset2 = ns1Id.dataset("ds2").toMetadataEntity();
final MetadataEntity program = appId.worker("wk1").toMetadataEntity(); final MetadataEntity dataset = nsId.dataset("ds2").toMetadataEntity();
@Test public void testUpdateSearch() throws IOException { MetadataStorage mds = getMetadataStorage(); final String ns = "ns"; final MetadataEntity program = new NamespaceId(ns).app("app1").worker("wk1").toMetadataEntity(); Metadata meta = new Metadata(USER, tags("tag1", "tag2"), props("key1", "value1", "key2", "value2")); MetadataRecord programRecord = new MetadataRecord(program, meta); mds.apply(new Update(program, meta)); assertResults(mds, SearchRequest.of("value1").addNamespace(ns).build(), programRecord); assertResults(mds, SearchRequest.of("value2").addNamespace(ns).build(), programRecord); assertResults(mds, SearchRequest.of("tag2").addNamespace(ns).build(), programRecord); mds.apply(new Update(program, new Metadata(USER, props("key1", "value3")))); mds.apply(new Remove(program, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, USER, "key2"), new ScopedNameOfKind(TAG, USER, "tag2")))); programRecord = new MetadataRecord(program, new Metadata(USER, tags("tag1"), props("key1", "value3"))); // Searching for value1 should be empty assertEmpty(mds, SearchRequest.of("value1").addNamespace(ns).build()); // Instead key1 has value value3 now assertResults(mds, SearchRequest.of("value3").addNamespace(ns).build(), programRecord); // key2 and tag2 were deleted assertEmpty(mds, SearchRequest.of("value2").addNamespace(ns).build()); assertEmpty(mds, SearchRequest.of("tag2").addNamespace(ns).build()); // tag1 is still here assertResults(mds, SearchRequest.of("tag1").addNamespace(ns).build(), programRecord); // clean up mds.apply(new Drop(program)); }
final MetadataEntity program = appId.worker("wk1").toMetadataEntity(); final MetadataEntity dataset = nsId.dataset("ds2").toMetadataEntity();
final NamespaceId ns2Id = new NamespaceId(ns2); final ApplicationId appId = ns1Id.app("app1"); final MetadataEntity program = appId.worker("wk1").toMetadataEntity();
NamespaceId nsId = new NamespaceId(ns); MetadataEntity service = nsId.app("app").service("service").toMetadataEntity(); MetadataEntity worker = nsId.app("app2").worker("worker").toMetadataEntity(); MetadataEntity dataset = nsId.dataset("dataset").toMetadataEntity(); MetadataEntity hidden = nsId.dataset("_auditLog").toMetadataEntity();