public TopologyStream(TopologyStream other) { if (other != null) { setId(other.getId()); setVersionId(other.getVersionId()); setStreamId(other.getStreamId()); setDescription(other.getDescription()); setTopologyId(other.getTopologyId()); if (other.getFields() != null) { setFields(other.getFields().stream().map(Field::copy).collect(Collectors.toList())); } setVersionTimestamp(other.getVersionTimestamp()); } }
public TopologyStream getStreamInfo(Long topologyId, Long streamId, Long versionId) { TopologyStream topologyStream = new TopologyStream(); topologyStream.setId(streamId); topologyStream.setVersionId(versionId); TopologyStream result = dao.get(new StorableKey(STREAMINFO_NAMESPACE, topologyStream.getPrimaryKey())); if (result == null || !result.getTopologyId().equals(topologyId)) { return null; } result.setVersionTimestamp(getVersionTimestamp(versionId)); return result; }
@Override public Map<String, Object> toMap() { Map<String, Object> map = super.toMap(); map.remove(FIELDS); try { map.put(FIELDSDATA, getFieldsData()); } catch (Exception e) { throw new RuntimeException(e); } return map; }
public Stream getStream(TopologyStream topologyStream) { return new Stream(topologyStream.getStreamId(), topologyStream.getFields()); }
public TopologyStream addStreamInfo(Long topologyId, Long versionId, TopologyStream topologyStream) { if (topologyStream.getId() == null) { topologyStream.setId(dao.nextId(STREAMINFO_NAMESPACE)); } long timestamp = System.currentTimeMillis(); topologyStream.setVersionTimestamp(timestamp); topologyStream.setVersionId(versionId); topologyStream.setTopologyId(topologyId); validateStreamInfo(topologyStream); dao.add(topologyStream); updateVersionTimestamp(versionId, timestamp); return topologyStream; }
private void validateStreamInfo(TopologyStream topologyStream) { if (topologyStream.getFields().isEmpty()) { throw new IllegalArgumentException("Stream with empty fields: " + topologyStream); } StorageUtils.ensureUnique(topologyStream, this::listStreamInfos, QueryParam.params(TopologyStream.TOPOLOGYID, topologyStream.getTopologyId().toString(), TopologyStream.VERSIONID, topologyStream.getVersionId().toString(), TopologyStream.STREAMID, topologyStream.getStreamId())); }
private void setReconfigureTarget(TopologyStream stream) { // reconfigure all targets of processors that use this output stream List<QueryParam> params = QueryParam.params( TopologyProcessorStreamMap.FIELD_VERSION_ID, String.valueOf(stream.getVersionId()), TopologyProcessorStreamMap.FIELD_STREAM_ID, String.valueOf(stream.getId())); listTopologyProcessorStreamMapping(params) .stream() .map(sm -> getTopologyProcessor(stream.getTopologyId(), sm.getProcessorId(), sm.getVersionId())) .flatMap(p -> getEdgesFrom(p).stream()) .filter(e -> e.getStreamGroupings().stream().map(StreamGrouping::getStreamId).anyMatch(sgid -> sgid.equals(stream.getId()))) .forEach(e -> setReconfigureTarget(e, stream)); // reconfigure all targets of source that use this output stream params = QueryParam.params( TopologySourceStreamMap.FIELD_VERSION_ID, String.valueOf(stream.getVersionId()), TopologySourceStreamMap.FIELD_STREAM_ID, String.valueOf(stream.getId())); listTopologySourceStreamMapping(params) .stream() .map(sm -> getTopologySource(stream.getTopologyId(), sm.getSourceId(), sm.getVersionId())) .flatMap(source -> getEdgesFrom(source).stream()) .filter(e -> e.getStreamGroupings().stream().map(StreamGrouping::getStreamId).anyMatch(sgid -> sgid.equals(stream.getId()))) .forEach(e -> setReconfigureTarget(e, stream)); }
private List<Long> importOutputStreams(Long newTopologyId, Map<Long, Long> oldToNewStreamIds, List<TopologyStream> streams) { List<Long> importedOutputStreamIds = new ArrayList<>(); for (TopologyStream stream : streams) { Long oldId = stream.getId(); Long newId = oldToNewStreamIds.get(oldId); if (newId == null) { stream.setId(null); TopologyStream addedTopologyStream = addStreamInfo(newTopologyId, stream); newId = addedTopologyStream.getId(); oldToNewStreamIds.put(oldId, newId); } importedOutputStreamIds.add(newId); } return importedOutputStreamIds; }
private List<Long> updateOutputStreams(TopologyOutputComponent outputComponent) { List<Long> newStreamIds = new ArrayList<>(); for (TopologyStream topologyStream : outputComponent.getOutputStreams()) { if (topologyStream.getId() != null && getStreamInfo(outputComponent.getTopologyId(), topologyStream.getId()) != null) { addOrUpdateStreamInfo(outputComponent.getTopologyId(), topologyStream.getId(), topologyStream); newStreamIds.add(topologyStream.getId()); } else { newStreamIds.add(addStreamInfo(outputComponent.getTopologyId(), topologyStream).getId()); } } return newStreamIds; }
public TopologyStream getStreamInfoByName(Long topologyId, String streamId, Long versionId) { List<QueryParam> queryParams = WSUtils.buildTopologyIdAndVersionIdAwareQueryParams(topologyId, versionId, null); try { for (TopologyStream topologyStream : listStreamInfos(queryParams)) { if (topologyStream.getStreamId().equals(streamId)) { return topologyStream; } } } catch (Exception ex) { LOG.error("Got exception ", ex); throw new RuntimeException(ex); } return null; }
public TopologyStream removeStreamInfo(Long topologyId, Long streamId, Long versionId) { TopologyStream topologyStream = getStreamInfo(topologyId, streamId, versionId); if (topologyStream != null) { setReconfigureTarget(topologyStream); topologyStream = dao.remove(new StorableKey(STREAMINFO_NAMESPACE, topologyStream.getPrimaryKey())); topologyStream.setVersionTimestamp(updateVersionTimestamp(versionId).getTimestamp()); } return topologyStream; }
Set<String> schemaFieldPatterns = getFieldPatterns( getStreamInfo(edge.getTopologyId(), streamGrouping.getStreamId(), edge.getVersionId()) .getFields()); fields.forEach(field -> { schemaFieldPatterns.stream().filter(pat -> field.matches(pat)).findAny()
private void updateWithSchemas(Long topologyId, Long versionId, Map<String, NormalizationConfig> normalizationConfigRead) { for (Map.Entry<String, NormalizationConfig> entry : normalizationConfigRead.entrySet()) { NormalizationConfig normalizationConfig = entry.getValue(); TopologyStream topologyStream = catalogService.getStreamInfoByName(topologyId, entry.getKey(), versionId); if (topologyStream != null) { normalizationConfig.setInputSchema(topologyStream.getSchema()); } } }
public TopologyStream addOrUpdateStreamInfo(Long topologyId, Long id, TopologyStream stream) { stream.setId(id); Long currentVersionId = getCurrentVersionId(topologyId); stream.setVersionId(currentVersionId); stream.setTopologyId(topologyId); long timestamp = System.currentTimeMillis(); stream.setVersionTimestamp(timestamp); validateStreamInfo(stream); TopologyStream curStream = getStreamInfo(topologyId, stream.getId()); if (!curStream.getFields().equals(stream.getFields())) { setReconfigureTarget(stream); } dao.addOrUpdate(stream); updateVersionTimestamp(currentVersionId, timestamp); return stream; }
private Map<String, Object> convertValueToConformStream(List<TopologyStream> outputStreams, String streamId, Map<String, Object> value) { Optional<TopologyStream> outputStream = outputStreams.stream().filter(o -> o.getStreamId().equals(streamId)).findFirst(); if (!outputStream.isPresent()) { throw new IllegalArgumentException("Stream " + streamId + " doesn't exist."); } return SchemaValueConverter.convertMap(Schema.of(outputStream.get().getFields()), value); }
private void setReconfigureTarget(TopologyEdge edge, TopologyStream stream) { TopologyComponent component = getTo(edge); component.setReconfigure(true); dao.addOrUpdate(component); // if component is a processor, update any rules in that processor that uses any of the streams if (component instanceof TopologyProcessor) { setReconfigureRules(Collections.singletonList((TopologyProcessor) component), edge.getStreamGroupings() .stream() .map(StreamGrouping::getStreamId) .map(sid -> getStreamInfo(edge.getTopologyId(), sid, edge.getVersionId())) .filter(curStream -> stream == null || curStream.getId().equals(stream.getId())) .collect(Collectors.toList())); } }
private Stream getStream(final String streamName) throws Exception { List<TopologyStream> topologyStreams = getStreamInfos().stream() .filter(s -> s.getStreamId().equalsIgnoreCase(streamName)) .collect(Collectors.toList()); if (topologyStreams.isEmpty()) { throw new IllegalArgumentException("Stream '" + streamName + "' does not exist"); } else if (topologyStreams.size() != 1) { throw new IllegalArgumentException("Stream '" + streamName + "' is not unique"); } else { TopologyStream topologyStream = topologyStreams.get(0); return new Stream(topologyStream.getStreamId(), topologyStream.getFields()); } }
private Collection<TopologySource> fillSourceStreams(Collection<TopologySource> sources) { if (sources != null) { for (TopologySource source : sources) { List<TopologyStream> topologyStreams = getOutputStreams(source); source.setOutputStreams(topologyStreams); source.setOutputStreamIds(new ArrayList<>(Collections2.transform(topologyStreams, new Function<TopologyStream, Long>() { @Nullable @Override public Long apply(@Nullable TopologyStream input) { return input.getId(); } }))); } } return sources; }
private Collection<TopologyProcessor> fillProcessorStreams(Collection<TopologyProcessor> processors) { if (processors != null) { for (TopologyProcessor processor : processors) { List<TopologyStream> topologyStreams = getOutputStreams(processor); processor.setOutputStreams(topologyStreams); processor.setOutputStreamIds(new ArrayList<>(Collections2.transform(topologyStreams, new Function<TopologyStream, Long>() { @Nullable @Override public Long apply(@Nullable TopologyStream input) { return input.getId(); } }))); } } return processors; }
private void createProcessorStreamMapping(TopologyProcessor topologyProcessor, List<TopologyStream> streams) { for (TopologyStream outputStream : streams) { TopologyStream addedStream = addStreamInfo(topologyProcessor.getTopologyId(), outputStream); dao.<TopologyProcessorStreamMap>add(new TopologyProcessorStreamMap(topologyProcessor.getId(), topologyProcessor.getVersionId(), addedStream.getId())); } }