private void outputMetrics(Nimbus.Iface client, Collection<String> names) throws Exception { ClusterSummary summary = client.getClusterInfo(); Set<String> ids = new HashSet<>(); for (TopologySummary ts: summary.get_topologies()) { long totalLatCount = 0; for (String id: ids) { TopologyInfo info = client.getTopologyInfo(id); @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") TopologyPageInfo tpi = client.getTopologyPageInfo(id, ":all-time", false); uptime = Math.max(uptime, info.get_uptime_secs()); for (ExecutorSummary exec : info.get_executors()) {
@Override public void close() { if (closed) { return; } RuntimeException saved = null; for (Iterator<String> it = super.iterator(); it.hasNext();) { String name = it.next(); try { client.killTopologyWithOpts(name, NO_WAIT_KILL); it.remove(); } catch (Exception e) { RuntimeException wrapped = new RuntimeException("Error trying to kill " + name, e); if (saved != null) { saved.addSuppressed(wrapped); } else { saved = wrapped; } } } super.clear(); if (saved != null) { throw saved; } closed = true; } }
@Override public void run(Nimbus.Iface nimbus) throws Exception { nimbus.activate(name); LOG.info("Activated topology: {}", name); } });
ClusterSummary stormCluster = nimbus.getClusterInfo(); int totalNumWorkers = stormCluster.get_supervisors().stream() .mapToInt(SupervisorSummary::get_num_workers).sum();
LOG.info("Capturing {}...", topologyName); String topologyId = topologySummary.get_id(); TopologyInfo info = client.getTopologyInfo(topologyId); TopologyPageInfo tpinfo = client.getTopologyPageInfo(topologyId, ":all-time", false); @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") StormTopology topo = client.getUserTopology(topologyId); Map<String, Object> topoConf = (Map<String, Object>) JSONValue.parse(client.getTopologyConf(topologyId)); for (String key: TopologyLoadConf.IMPORTANT_CONF_KEYS) { Object o = topoConf.get(key);
final String topoId = containerRequestContext.getUriInfo().getPathParameters().get("id").get(0); try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf)){ topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(topoId)); } catch (AuthorizationException ae) { LOG.error("Nimbus isn't allowing {} to access the topology conf of {}. {}", ReqContext.context(), topoId, ae.get_msg());
ClusterSummary summary = client.getClusterInfo(); for (TopologySummary topo : summary.get_topologies()) { if (topologyName.equals(topo.get_name())) { topologyConf = (Map<String, Object>) JSONValue.parse(client.getTopologyConf(topologyId)); LOG.info("Using topology conf from {} as basis for getting new creds", topologyId);
List<String> topologyNames = cmd.getArgList(); ClusterSummary clusterSummary = client.getClusterInfo(); for (TopologySummary topologySummary: clusterSummary.get_topologies()) { if (topologyNames.isEmpty() || topologyNames.contains(topologySummary.get_name())) {
ClusterSummary clusterSummary = client.getClusterInfo(); TopologySummary topologySummary = null; for (TopologySummary ts : clusterSummary.get_topologies()) { GetInfoOptions getInfoOpts = new GetInfoOptions(); getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE); TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts); for (ExecutorSummary es : info.get_executors()) { if (_component.equals(es.get_component_id())) {
user.set(new ReqContext(ReqContext.context())); return null; }).when(impl).activate(anyString()); client.getClient().activate("bad_auth_test_topology"); fail("We should not be able to connect without a token..."); } catch (Exception e) { verify(impl, times(2)).activate("topo-bob"); verify(impl, times(3)).activate("topo-alice"); verify(impl, never()).activate("bad_auth_test_topology"); LOG.info("\n\n\t\tworkerTokenDigestAuthTest - END\n\n");
List<String> topologyNames = cmd.getArgList(); ClusterSummary clusterSummary = client.getClusterInfo(); for (TopologySummary topologySummary: clusterSummary.get_topologies()) { if (topologyNames.isEmpty() || topologyNames.contains(topologySummary.get_name())) {
rbm = client.getClient().getBlobMeta(key); remoteBlobStore = new NimbusBlobStore(); remoteBlobStore.setClient(conf, client);
user.set(new ReqContext(ReqContext.context())); return null; }).when(impl).activate(anyString()); (ThriftServer server, Map<String, Object> conf) -> { try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) { client.getClient().activate("security_auth_test_topology"); badTransport.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, SimpleTransportPlugin.class.getName()); try (NimbusClient client = new NimbusClient(badTransport, "localhost", server.getPort(), NIMBUS_TIMEOUT)) { client.getClient().activate("bad_security_auth_test_topology"); fail("An exception should have been thrown trying to connect."); } catch (Exception te) { verifyIncorrectJaasConf(server, conf, MISSING_CLIENT, IOException.class); }); verify(impl).activate("security_auth_test_topology"); verify(impl, never()).activate("bad_auth_test_topology");
public static void printMetrics(Nimbus.Iface client, String name) throws Exception { ClusterSummary summary = client.getClusterInfo(); String id = null; for (TopologySummary ts : summary.get_topologies()) { throw new Exception("Could not find a topology named " + name); TopologyInfo info = client.getTopologyInfo(id); int uptime = info.get_uptime_secs(); long acked = 0;
public static void printMetrics(Nimbus.Iface client, String name) throws Exception { ClusterSummary summary = client.getClusterInfo(); String id = null; for (TopologySummary ts : summary.get_topologies()) { throw new Exception("Could not find a topology named " + name); TopologyInfo info = client.getTopologyInfo(id); int uptime = info.get_uptime_secs(); long acked = 0;
static void printMetrics(Nimbus.Iface client, String name) throws Exception { ClusterSummary summary = client.getClusterInfo(); String id = null; for (TopologySummary ts : summary.get_topologies()) { throw new Exception("Could not find a topology named " + name); TopologyInfo info = client.getTopologyInfo(id); int uptime = info.get_uptime_secs(); long acked = 0;
@Test public void simpleAuthTest() throws Exception { Nimbus.Iface impl = mock(Nimbus.Iface.class); withServer(SimpleTransportPlugin.class, impl, (ThriftServer server, Map<String, Object> conf) -> { try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) { client.getClient().activate("security_auth_test_topology"); } //Verify digest is rejected... Map<String, Object> badConf = new HashMap<>(conf); badConf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, DigestSaslTransportPlugin.class.getName()); badConf.put("java.security.auth.login.config", DIGEST_JAAS_CONF); badConf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0); try (NimbusClient client = new NimbusClient(badConf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) { client.getClient().activate("bad_security_auth_test_topology"); fail("An exception should have been thrown trying to connect."); } catch (Exception te) { LOG.info("Got Exception...", te); assert (Utils.exceptionCauseIsInstanceOf(TTransportException.class, te)); } }); verify(impl).activate("security_auth_test_topology"); verify(impl, never()).activate("bad_security_auth_test_topology"); }
private HashSet<String> getComponents(Nimbus.Iface client, String topology) throws Exception { HashSet<String> components = new HashSet<>(); ClusterSummary clusterSummary = client.getClusterInfo(); TopologySummary topologySummary = null; for (TopologySummary ts : clusterSummary.get_topologies()) { if (topology.equals(ts.get_name())) { topologySummary = ts; break; } } if (topologySummary == null) { throw new IllegalArgumentException("topology: " + topology + " not found"); } else { String id = topologySummary.get_id(); GetInfoOptions getInfoOpts = new GetInfoOptions(); getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE); TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts); for (ExecutorSummary es : info.get_executors()) { components.add(es.get_component_id()); } } return components; }
/** * Get a specific owner resource summary. * @param ownerResourceSummaries Result from thrift call. * @param client client * @param id Owner id. * @param config Storm conf. * @return prettified owner resource summary. */ public static Map<String, Object> getOwnerResourceSummary( List<OwnerResourceSummary> ownerResourceSummaries, Nimbus.Iface client, String id, Map<String, Object> config) throws TException { Map<String, Object> result = new HashMap(); result.put("schedulerDisplayResource", config.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE)); if (ownerResourceSummaries.isEmpty()) { return unpackOwnerResourceSummary(new OwnerResourceSummary(id)); } List<TopologySummary> topologies = null; topologies = client.getClusterInfo().get_topologies(); List<Map> topologySummaries = getTopologiesMap(id, topologies); result.putAll(unpackOwnerResourceSummary(ownerResourceSummaries.get(0))); result.put("topologies", topologySummaries); return result; }
/** * /api/v1/topology/summary -> topo history. */ @GET @Path("/topology/summary") @AuthNimbusOp("getClusterInfo") @Produces("application/json") public Response getTopologySummary(@Context SecurityContext securityContext, @QueryParam(callbackParameterName) String callback) throws TException { allTopologiesSummaryRequestMeter.mark(); try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(config)) { return UIHelpers.makeStandardResponse( UIHelpers.getAllTopologiesSummary( nimbusClient.getClient().getClusterInfo().get_topologies(), config ), callback ); } }