public static void submitRebalance(String topologyName, RebalanceOptions options, Map conf) throws Exception { Map stormConf = Utils.readStormConfig(); if (conf != null) { stormConf.putAll(conf); } NimbusClient client = null; try { client = NimbusClient.getConfiguredClient(stormConf); client.getClient().rebalance(topologyName, options); } catch (Exception e) { throw e; } finally { if (client != null) { client.close(); } } }
public static NimbusClient getConfiguredClient(Map conf) { String nimbusHost = (String) conf.get(Config.NIMBUS_HOST); int nimbusPort = Utils.getInt(conf.get(Config.NIMBUS_THRIFT_PORT)); return new NimbusClient(nimbusHost, nimbusPort); }
public static NimbusClient getConfiguredClientAs(Map conf, Integer timeout, String asUser) { try { if (conf.containsKey(Config.STORM_DO_AS_USER)) { if (asUser != null && !asUser.isEmpty()) { LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, " + "config will take precedence.", asUser, conf.get(Config.STORM_DO_AS_USER)); } asUser = (String) conf.get(Config.STORM_DO_AS_USER); } NimbusClient client = new NimbusClient(conf, null, null, timeout, asUser); checkVersion(client); return client; } catch (Exception ex) { throw new RuntimeException(ex); } }
public static void downloadFromMaster(Map conf, String file, String localFile) throws IOException, TException { NimbusClient client = NimbusClient.getConfiguredClient(conf); String id = client.getClient().beginFileDownload(file); WritableByteChannel out = Channels.newChannel(new FileOutputStream( localFile)); while (true) { ByteBuffer chunk = client.getClient().downloadChunk(id); int written = out.write(chunk); if (written == 0) break; } out.close(); }
public static void checkVersion(NimbusClient client) { String serverVersion; try { serverVersion = client.getClient().getVersion(); } catch (TException e) { LOG.warn("Failed to get nimbus version "); return; } if (!clientVersion.equals(serverVersion)) { LOG.warn("Your client version: " + clientVersion + " but nimbus version: " + serverVersion); } }
public static List<String> getSupervisorHosts() throws Exception { try { List<String> hosts = new ArrayList<>(); NimbusClient client = getNimbusClient(null); ClusterSummary clusterSummary = client.getClient().getClusterInfo(); List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors(); Collections.sort(supervisorSummaries, new Comparator<SupervisorSummary>() { @Override public int compare(SupervisorSummary o1, SupervisorSummary o2) { int o1Left = o1.get_numWorkers() - o1.get_numUsedWorkers(); int o2Left = o2.get_numWorkers() - o2.get_numUsedWorkers(); return o1Left - o2Left; } }); for (SupervisorSummary supervisorSummary : supervisorSummaries) { hosts.add(supervisorSummary.get_host()); } return hosts; } catch (Exception e) { if (client != null) { client.close(); client = null; } LOG.error("Failed to kill all topologies ", e); throw new RuntimeException(e); } }
@SuppressWarnings("unchecked") public static NimbusClient getConfiguredClient(Map conf) { return getConfiguredClient(conf, null); }
client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null); rbm = client.getClient().getBlobMeta(key); remoteBlobStore = new NimbusBlobStore(); remoteBlobStore.setClient(conf, client);
@Override public void shutdown() { if (client != null) { client.close(); client = null; } } }
@Override public void init(Map conf) throws Exception { this.conf = conf; isLocal = StormConfig.try_local_mode(conf); if (isLocal) { client = LocalCluster.getInstance().getLocalClusterMap().getNimbus(); } else { remoteClient = NimbusClient.getConfiguredClient(conf); client = remoteClient.getClient(); } isValid.set(true); }
@Override public void cancel() throws IOException { try { synchronized(client) { client.getClient().cancelBlobUpload(session); } } catch (TException e) { throw new RuntimeException(e); } }
public static void cleanCluster() { try { NimbusClient client = getNimbusClient(null); ClusterSummary clusterSummary = client.getClient().getClusterInfo(); List<TopologySummary> topologySummaries = clusterSummary.get_topologies(); KillOptions killOption = new KillOptions(); killOption.set_wait_secs(1); for (TopologySummary topologySummary : topologySummaries) { client.getClient().killTopologyWithOpts(topologySummary.get_name(), killOption); LOG.info("Successfully kill " + topologySummary.get_name()); } } catch (Exception e) { if (client != null) { client.close(); client = null; } LOG.error("Failed to kill all topology ", e); } }
@Override public void prepare(Map conf) { this.client = NimbusClient.getConfiguredClient(conf); if (conf != null) { this.bufferSize = JStormUtils.parseInt(conf.get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize); } }
client = new NimbusClient(conf, nimbusInfo.getHost(), nimbusInfo.getPort(), null); rbm = client.getClient().getBlobMeta(key); remoteBlobStore = new NimbusBlobStore(); remoteBlobStore.setClient(conf, client);
@Override public void cleanup() { invalidate(); if (remoteClient != null) { remoteClient.close(); } }
public static NimbusClient getConfiguredClientAs(Map conf, Integer timeout, String asUser) { try { if (conf.containsKey(Config.STORM_DO_AS_USER)) { if (asUser != null && !asUser.isEmpty()) { LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, config will take precedence.", asUser, conf.get(Config.STORM_DO_AS_USER)); } asUser = (String) conf.get(Config.STORM_DO_AS_USER); } NimbusClient client = new NimbusClient(conf, null, null, timeout, asUser); checkVersion(client); return client; } catch (Exception ex) { throw new RuntimeException(ex); } }
public static void downloadFromMaster(Map conf, String file, String localFile) throws IOException, TException { WritableByteChannel out = null; NimbusClient client = null; try { client = NimbusClient.getConfiguredClient(conf, 10 * 1000); String id = client.getClient().beginFileDownload(file); out = Channels.newChannel(new FileOutputStream(localFile)); while (true) { ByteBuffer chunk = client.getClient().downloadChunk(id); int written = out.write(chunk); if (written == 0) { client.getClient().finishFileDownload(id); break; } } } finally { if (out != null) out.close(); if (client != null) client.close(); } }
public static void downloadFromMaster(Map conf, String file, String localFile) throws IOException, TException { NimbusClient client = NimbusClient.getConfiguredClient(conf); String id = client.getClient().beginFileDownload(file); WritableByteChannel out = Channels.newChannel(new FileOutputStream(localFile)); while(true) { ByteBuffer chunk = client.getClient().downloadChunk(id); int written = out.write(chunk); if(written==0) break; } out.close(); }
@Override public int updateBlobReplication(String key, int replication) throws KeyNotFoundException { try { return client.getClient().updateBlobReplication(key, replication); } catch (KeyNotFoundException exp) { throw exp; } catch (TException e) { throw new RuntimeException(e); } }
public static NimbusClient getNimbusClient(Map conf) { if (client != null) { return client; } if (conf == null) { conf = Utils.readStormConfig(); } client = NimbusClient.getConfiguredClient(conf); return client; }