public TopologyProcessor getTopologyProcessor(Long topologyId, Long processorId, Long versionId) { TopologyProcessor topologyProcessor = new TopologyProcessor(); topologyProcessor.setId(processorId); topologyProcessor.setVersionId(versionId); TopologyProcessor processor = dao.get(new StorableKey(TOPOLOGY_PROCESSOR_NAMESPACE, topologyProcessor.getPrimaryKey())); if (processor == null || !processor.getTopologyId().equals(topologyId)) { return null; } fillProcessorStreams(processor); processor.setVersionTimestamp(getVersionTimestamp(versionId)); return processor; }
public TopologyProcessor addTopologyProcessor(Long topologyId, Long versionId, TopologyProcessor topologyProcessor) { if (topologyProcessor.getId() == null) { topologyProcessor.setId(getNextTopologyComponentId()); } topologyProcessor.setVersionId(versionId); topologyProcessor.setTopologyId(topologyId); validateTopologyProcessor(topologyProcessor); List<TopologyStream> topologyStreams = addTopologyOutputComponent(topologyProcessor); addProcessorStreamMapping(topologyProcessor, topologyProcessor.getOutputStreamIds()); topologyProcessor.setOutputStreams(topologyStreams); topologyProcessor.setVersionTimestamp(updateVersionTimestamp(versionId).getTimestamp()); return topologyProcessor; }
private void validateTopologyProcessor(TopologyProcessor topologyProcessor) { StorageUtils.ensureUnique(topologyProcessor, this::listTopologyProcessors, QueryParam.params(TopologyProcessor.TOPOLOGYID, topologyProcessor.getTopologyId().toString(), TopologyProcessor.VERSIONID, topologyProcessor.getVersionId().toString(), TopologyProcessor.NAME, topologyProcessor.getName())); }
private void addProcessorStreamMapping(TopologyProcessor topologyProcessor, List<Long> streamIds) { for (Long outputStreamId : streamIds) { dao.<TopologyProcessorStreamMap>add(new TopologyProcessorStreamMap(topologyProcessor.getId(), topologyProcessor.getVersionId(), outputStreamId)); } }
topologyProcessor.setOutputStreamIds(importOutputStreams(newTopology.getId(), oldToNewStreamIds, topologyProcessor.getOutputStreams())); topologyProcessor.setOutputStreams(null); Long oldComponentId = topologyProcessor.getId(); topologyProcessor.setId(null); topologyProcessor.setTopologyId(newTopology.getId()); TopologyComponentBundle bundle; String subType = topologyData.getBundleIdToType().get(topologyProcessor.getTopologyComponentBundleId().toString()); if (TopologyLayoutConstants.JSON_KEY_CUSTOM_PROCESSOR_SUB_TYPE.equals(subType)) { QueryParam queryParam = new QueryParam(CustomProcessorInfo.NAME, topologyProcessor.getConfig().get(CustomProcessorInfo.NAME)); Collection<TopologyComponentBundle> result = listCustomProcessorBundlesWithFilter(Collections.singletonList(queryParam)); if (result.size() != 1) { throw new IllegalStateException("Not able to find topology component bundle for custom processor :" + topologyProcessor.getConfig().get (CustomProcessorInfo.NAME)); bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, subType); topologyProcessor.setTopologyComponentBundleId(bundle.getId()); Optional<Object> ruleListObj = topologyProcessor.getConfig().getAnyOptional(RulesProcessor.CONFIG_KEY_RULES); ruleListObj.ifPresent(ruleList -> { List<Long> ruleIds = new ObjectMapper().convertValue(ruleList, new TypeReference<List<Long>>() {}); topologyProcessor.getConfig().setAny(RulesProcessor.CONFIG_KEY_RULES, updatedRuleIds); }); addTopologyProcessor(newTopology.getId(), topologyProcessor); oldToNewComponentIds.put(oldComponentId, topologyProcessor.getId());
public TopologyProcessor addOrUpdateTopologyProcessor(Long topologyId, Long id, TopologyProcessor topologyProcessor) { Long currentTopologyVersionId = getCurrentVersionId(topologyId); topologyProcessor.setId(id); topologyProcessor.setVersionId(currentTopologyVersionId); topologyProcessor.setTopologyId(topologyId); validateTopologyProcessor(topologyProcessor); topologyProcessor.setReconfigure(false); dao.addOrUpdate(topologyProcessor); List<Long> newList = Collections.emptyList(); if (topologyProcessor.getOutputStreamIds() != null) { newList = topologyProcessor.getOutputStreamIds(); } else if (topologyProcessor.getOutputStreams() != null) { newList = updateOutputStreams(topologyProcessor); } List<Long> existing = getOutputStreamIds(topologyProcessor); Sets.SetView<Long> streamIdsToRemove = Sets.difference(ImmutableSet.copyOf(existing), ImmutableSet.copyOf(newList)); Sets.SetView<Long> streamIdsToAdd = Sets.difference(ImmutableSet.copyOf(newList), ImmutableSet.copyOf(existing)); removeProcessorStreamMapping(topologyProcessor, Lists.newArrayList(streamIdsToRemove)); addProcessorStreamMapping(topologyProcessor, Lists.newArrayList(streamIdsToAdd)); TopologyProcessor updatedProcessor = getTopologyProcessor(topologyId, id, currentTopologyVersionId); updatedProcessor.setVersionTimestamp(updateVersionTimestamp(currentTopologyVersionId).getTimestamp()); return topologyProcessor; }
private void setReconfigureRules(List<TopologyProcessor> processors, List<TopologyStream> affectedStreams) { Map<Long, BiFunction<TopologyProcessor, Long, BaseTopologyRule>> bundles = new HashMap<>(); TopologyComponentBundle bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.RULE); bundles.put(bundle.getId(), (p, r) -> getRule(p.getTopologyId(), r, p.getVersionId())); bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.BRANCH); bundles.put(bundle.getId(), (p, r) -> getBranchRule(p.getTopologyId(), r, p.getVersionId())); bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.PROJECTION); bundles.put(bundle.getId(), (p, r) -> getRule(p.getTopologyId(), r, p.getVersionId())); bundle = getCurrentTopologyComponentBundle(TopologyComponentBundle.TopologyComponentType.PROCESSOR, ComponentTypes.WINDOW); bundles.put(bundle.getId(), (p, r) -> getWindow(p.getTopologyId(), r, p.getVersionId())); if ((function = bundles.get(processor.getTopologyComponentBundleId())) != null) { Optional<Object> ruleList = processor.getConfig().getAnyOptional(RulesProcessor.CONFIG_KEY_RULES); if (ruleList.isPresent()) { ObjectMapper objectMapper = new ObjectMapper();
private void createProcessorStreamMapping(TopologyProcessor topologyProcessor, List<TopologyStream> streams) { for (TopologyStream outputStream : streams) { TopologyStream addedStream = addStreamInfo(topologyProcessor.getTopologyId(), outputStream); dao.<TopologyProcessorStreamMap>add(new TopologyProcessorStreamMap(topologyProcessor.getId(), topologyProcessor.getVersionId(), addedStream.getId())); } }
public TopologyProcessor removeTopologyProcessor(Long topologyId, Long processorId, Long versionId, boolean removeEdges) { TopologyProcessor topologyProcessor = getTopologyProcessor(topologyId, processorId, versionId); if (topologyProcessor != null) { if (removeEdges) { removeAllEdges(topologyProcessor); } removeProcessorStreamMapping(topologyProcessor); topologyProcessor = dao.<TopologyProcessor>remove(new StorableKey(TOPOLOGY_PROCESSOR_NAMESPACE, topologyProcessor.getPrimaryKey())); topologyProcessor.setVersionTimestamp(updateVersionTimestamp(versionId).getTimestamp()); } return topologyProcessor; }
private void removeProcessorStreamMapping(TopologyProcessor topologyProcessor) { if (topologyProcessor != null) { removeProcessorStreamMapping(topologyProcessor, topologyProcessor.getOutputStreamIds()); } }
public CustomProcessorInfo removeCustomProcessorInfoAsBundle(String name) throws IOException { List<QueryParam> queryParams = new ArrayList<>(); queryParams.add(new QueryParam(CustomProcessorInfo.NAME, name)); Collection<TopologyComponentBundle> result = this.listCustomProcessorBundlesWithFilter(queryParams); if (result.isEmpty() || result.size() != 1) { throw new IOException("Failed to delete custom processor with name:" + name); } TopologyComponentBundle customProcessorBundle = result.iterator().next(); Collection<TopologyProcessor> processors = this.listTopologyProcessors(); if (processors != null && !processors.isEmpty()) { for (TopologyProcessor topologyProcessor: processors) { if (topologyProcessor.getTopologyComponentBundleId().equals(customProcessorBundle.getId())) { throw new IOException("Cannot delete custom processor as it is being used in one of the topologies."); } } } this.removeTopologyComponentBundle(customProcessorBundle.getId()); return CustomProcessorInfo.fromTopologyComponentBundle(customProcessorBundle); }
removeTopologyProcessor(topologyId, processor.getId(), versionId, false);
addTopologyProcessor(topologyId, newVersionId, new TopologyProcessor(processor));
private List<TopologyStream> getOutputStreams(TopologyProcessor topologyProcessor) { List<TopologyStream> streams = new ArrayList<>(); if (topologyProcessor != null) { QueryParam qp1 = new QueryParam(TopologyProcessorStreamMap.FIELD_PROCESSOR_ID, String.valueOf(topologyProcessor.getId())); QueryParam qp2 = new QueryParam(TopologyProcessorStreamMap.FIELD_VERSION_ID, String.valueOf(topologyProcessor.getVersionId())); for (TopologyProcessorStreamMap mapping : listTopologyProcessorStreamMapping(ImmutableList.of(qp1, qp2))) { TopologyStream topologyStream = getStreamInfo(topologyProcessor.getTopologyId(), mapping.getStreamId(), topologyProcessor.getVersionId()); if (topologyStream != null) { streams.add(topologyStream); } } } return streams; }
private void removeProcessorStreamMapping(TopologyProcessor topologyProcessor, List<Long> streamIds) { if (topologyProcessor != null) { for (Long outputStreamId : streamIds) { TopologyProcessorStreamMap mapping = new TopologyProcessorStreamMap(topologyProcessor.getId(), topologyProcessor.getVersionId(), outputStreamId); dao.<TopologyProcessorStreamMap>remove(mapping.getStorableKey()); } } }
private List<Long> getOutputStreamIds(TopologyProcessor topologyProcessor) { List<Long> streamIds = new ArrayList<>(); if (topologyProcessor != null) { QueryParam qp1 = new QueryParam(TopologyProcessorStreamMap.FIELD_PROCESSOR_ID, String.valueOf(topologyProcessor.getId())); QueryParam qp2 = new QueryParam(TopologyProcessorStreamMap.FIELD_VERSION_ID, String.valueOf(topologyProcessor.getVersionId())); for (TopologyProcessorStreamMap mapping : listTopologyProcessorStreamMapping(ImmutableList.of(qp1, qp2))) { streamIds.add(mapping.getStreamId()); } } return streamIds; }