public String setUpExtraJars(Topology topology, TopologyActions topologyActions) throws IOException { StormTopologyDependenciesHandler extraJarsHandler = new StormTopologyDependenciesHandler(catalogService); topology.getTopologyDag().traverse(extraJarsHandler); Path extraJarsLocation = topologyActions.getExtraJarsLocation(CatalogToLayoutConverter.getTopologyLayout(topology)); makeEmptyDir(extraJarsLocation); Set<String> extraJars = new HashSet<>(); extraJars.addAll(extraJarsHandler.getExtraJars()); extraJars.addAll(getBundleJars(extraJarsHandler.getTopologyComponentBundleSet())); downloadAndCopyJars(extraJars, extraJarsLocation); return extraJarsHandler.getMavenDeps(); }
public TopologyTestRunHistory runTest(TopologyActions topologyActions, Topology topology, TopologyTestRunCase testCase, Long durationSecs) throws IOException { List<StreamlineSource> sources = topology.getTopologyDag().getOutputComponents().stream() .filter(c -> c instanceof StreamlineSource) .map(c -> (StreamlineSource) c) .collect(toList()); List<StreamlineSink> sinks = topology.getTopologyDag().getInputComponents().stream() .filter(c -> c instanceof StreamlineSink) .map(c -> (StreamlineSink) c) .collect(toList()); List<StreamlineProcessor> processors = topology.getTopologyDag().getOutputComponents().stream() .filter(c -> c instanceof StreamlineProcessor && !(c instanceof RulesProcessor)) .map(c -> (StreamlineProcessor) c) .collect(toList()); List<RulesProcessor> rulesProcessors = topology.getTopologyDag().getOutputComponents().stream() .filter(c -> c instanceof RulesProcessor) .map(c -> (RulesProcessor) c)
@Override public void deploy(TopologyContext context) throws Exception { TopologyActions topologyActions = context.getTopologyActions(); Topology topology = context.getTopology(); TopologyDag dag = topology.getTopologyDag(); TopologyLayout layout = CatalogToLayoutConverter.getTopologyLayout(topology, dag); if (dag == null) { throw new IllegalStateException("Topology dag not set up"); } try { context.setCurrentAction("Submitting topology to streaming engine"); String mavenArtifacts = context.getMavenArtifacts(); topologyActions.deploy(layout, mavenArtifacts, context, context.getAsUser()); context.setState(TOPOLOGY_STATE_DEPLOYED); context.setCurrentAction("Topology deployed"); } catch (Exception ex) { LOG.error("Error while trying to deploy the topology in the streaming engine", ex); LOG.error("Trying to kill any running instance of topology '{}'", context.getTopology().getName()); killTopologyIfRunning(context, layout); context.setState(TOPOLOGY_STATE_DEPLOYMENT_FAILED); context.setCurrentAction("Topology submission failed due to: " + ex); throw new IgnoreTransactionRollbackException(ex); } }
@Override public void deploy(TopologyContext context) throws Exception { try { context.setCurrentAction("Validating topology DAG"); TopologyDag dag = context.getTopology().getTopologyDag(); context.getTopologyActionsService().ensureValid(dag); context.setState(TOPOLOGY_STATE_DAG_VALIDATED); context.setCurrentAction("Topology DAG validated"); } catch (Exception ex) { context.setState(TOPOLOGY_STATE_DEPLOYMENT_FAILED); context.setCurrentAction("Topology DAG validation failed due to: " + ex); throw new IgnoreTransactionRollbackException(ex); } } };
private Void runTestInBackground(TopologyActions topologyActions, Topology topology, TopologyTestRunHistory history, Map<String, TestRunSource> testRunSourceMap, Map<String, TestRunProcessor> testRunProcessorMap, Map<String, TestRunRulesProcessor> testRunRulesProcessorMap, Map<String, TestRunSink> testRunSinkMap, Map<String, List<Map<String, Object>>> expectedOutputRecordsMap, Optional<Long> durationSecs) throws IOException { TopologyLayout topologyLayout = CatalogToLayoutConverter.getTopologyLayout(topology, topology.getTopologyDag()); try { topologyActionsService.setUpClusterArtifacts(topology, topologyActions); String mavenArtifacts = topologyActionsService.setUpExtraJars(topology, topologyActions); topologyActions.runTest(topologyLayout, history, mavenArtifacts, testRunSourceMap, testRunProcessorMap, testRunRulesProcessorMap, testRunSinkMap, durationSecs); history.finishSuccessfully(); Map<String, List<Map<String, Object>>> actualOutputRecordsMap = parseTestRunOutputFiles(testRunSinkMap); history.setActualOutputRecords(objectMapper.writeValueAsString(actualOutputRecordsMap)); if (expectedOutputRecordsMap != null) { boolean matched = equalsOutputRecords(expectedOutputRecordsMap, actualOutputRecordsMap); history.setMatched(matched); } } catch (Throwable e) { LOG.warn("Exception thrown while running Topology as test mode. Marking as 'failed'. topology id: {}", topology.getId(), e); history.finishWithFailures(); } catalogService.addOrUpdateTopologyTestRunHistory(history.getId(), history); return null; }
private TopologyData doExportTopology(Topology topology) throws Exception { TopologyDag dag = topologyDagBuilder.getDag(topology); topology.setTopologyDag(dag); TopologyData topologyData = new TopologyData(); TopologyExportVisitor exportVisitor = new TopologyExportVisitor(topology.getId(), topologyData, this); topologyData.setTopologyName(topology.getName()); topologyData.setConfig(topology.getConfig()); TopologyDag topologyDag = topology.getTopologyDag(); if (topologyDag != null) { topologyDag.traverse(exportVisitor); } topologyData.setMetadata(getTopologyEditorMetadata(topology.getId())); return topologyData; }