public List<ACL> getDefaultZkAcls() { return daemonType.getDefaultZkAcls(conf); }
public List<ACL> getZkSecretAcls(WorkerTokenServiceType type) { return daemonType.getZkSecretAcls(type, conf); } }
/** * Get the ACLs to store a secret for a given service. * * @param type the type of service the secret is for. * @param conf the cluster configuration. * @return the ACLs */ public List<ACL> getZkSecretAcls(WorkerTokenServiceType type, Map<String, Object> conf) { throw new IllegalArgumentException(name() + " does not support storing secrets."); } }
public ZKStateStorage(Map<String, Object> conf, Map<String, Object> authConf, ClusterStateContext context) throws Exception { this.conf = conf; this.authConf = authConf; if (context.getDaemonType().equals(DaemonType.NIMBUS)) { this.isNimbus = true; } // just mkdir STORM_ZOOKEEPER_ROOT dir CuratorFramework zkTemp = mkZk(context.getDaemonType()); String rootPath = String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT)); ClientZookeeper.mkdirs(zkTemp, rootPath, context.getDefaultZkAcls()); zkTemp.close(); active = new AtomicBoolean(true); zkWriter = mkZk(new ZkWatcherCallBack(), context.getDaemonType()); if (isNimbus) { zkReader = mkZk(new ZkWatcherCallBack(), context.getDaemonType()); } else { zkReader = zkWriter; } }
public static void start(Map<String, Object> stormConfig, DaemonType type) { try { hostName = dotToUnderScore(Utils.localHostname()); } catch (UnknownHostException e) { LOG.warn("Unable to determine hostname while starting the metrics system. Hostname will be reported" + " as 'localhost'."); } LOG.info("Starting metrics reporters..."); List<Map<String, Object>> reporterList = (List<Map<String, Object>>) stormConfig.get(Config.STORM_METRICS_REPORTERS); if (reporterList != null && reporterList.size() > 0) { for (Map<String, Object> reporterConfig : reporterList) { // only start those requested List<String> daemons = (List<String>) reporterConfig.get("daemons"); for (String daemon : daemons) { if (DaemonType.valueOf(daemon.toUpperCase()) == type) { startReporter(stormConfig, reporterConfig); } } } } }
public ZKStateStorage(Map<Object, Object> conf, Map authConf, List<ACL> acls, ClusterStateContext context) throws Exception { this.conf = conf; this.authConf = authConf; if (context.getDaemonType().equals(DaemonType.NIMBUS)) this.isNimbus = true; // just mkdir STORM_ZOOKEEPER_ROOT dir CuratorFramework zkTemp = mkZk(acls); String rootPath = String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT)); Zookeeper.mkdirs(zkTemp, rootPath, acls); zkTemp.close(); active = new AtomicBoolean(true); zkWriter = mkZk(acls, new ZkWatcherCallBack()); if (isNimbus) { zkReader = mkZk(acls, new ZkWatcherCallBack()); } else { zkReader = zkWriter; } }
public static void start(Map<String, Object> stormConfig, DaemonType type){ try { hostName = dotToUnderScore(Utils.localHostname()); } catch (UnknownHostException e) { LOG.warn("Unable to determine hostname while starting the metrics system. Hostname will be reported" + " as 'localhost'."); } LOG.info("Starting metrics reporters..."); List<Map<String, Object>> reporterList = (List<Map<String, Object>>)stormConfig.get(Config.STORM_METRICS_REPORTERS); if(reporterList != null && reporterList.size() > 0) { for (Map<String, Object> reporterConfig : reporterList) { // only start those requested List<String> daemons = (List<String>) reporterConfig.get("daemons"); for (String daemon : daemons) { if (DaemonType.valueOf(daemon.toUpperCase()) == type) { startReporter(stormConfig, reporterConfig); } } } } }
public static CuratorFramework createZKClient(Map<String, Object> conf, DaemonType type) { @SuppressWarnings("unchecked") List<String> zkServers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); Object port = conf.get(Config.STORM_ZOOKEEPER_PORT); ZookeeperAuthInfo zkAuthInfo = new ZookeeperAuthInfo(conf); CuratorFramework zkClient = CuratorUtils.newCurator(conf, zkServers, port, (String) conf.get(Config.STORM_ZOOKEEPER_ROOT), zkAuthInfo, type.getDefaultZkAcls(conf)); zkClient.start(); return zkClient; }
@Test public void getDefaultZkAclsSecureServerConf() { Map<String, Object> conf = ConfigUtils.readStormConfig(); conf.put(Config.STORM_ZOOKEEPER_AUTH_SCHEME, "digest"); conf.put(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD, "storm:thisisapoorpassword"); conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); conf.put(Config.NIMBUS_THRIFT_PORT, 6666); assertNull(DaemonType.UNKNOWN.getDefaultZkAcls(conf)); assertNull(DaemonType.PACEMAKER.getDefaultZkAcls(conf)); assertEquals(DaemonType.NIMBUS_SUPERVISOR_ZK_ACLS, DaemonType.SUPERVISOR.getDefaultZkAcls(conf)); assertEquals(DaemonType.NIMBUS_SUPERVISOR_ZK_ACLS, DaemonType.NIMBUS.getDefaultZkAcls(conf)); assertNull(DaemonType.WORKER.getDefaultZkAcls(conf)); }
@Test public void getDefaultZkAclsDefaultConf() { Map<String, Object> conf = ConfigUtils.readStormConfig(); assertNull(DaemonType.UNKNOWN.getDefaultZkAcls(conf)); assertNull(DaemonType.PACEMAKER.getDefaultZkAcls(conf)); assertNull(DaemonType.SUPERVISOR.getDefaultZkAcls(conf)); assertNull(DaemonType.NIMBUS.getDefaultZkAcls(conf)); assertNull(DaemonType.WORKER.getDefaultZkAcls(conf)); }
protected TransactionalState(Map<String, Object> conf, String id, String subroot) { try { conf = new HashMap<>(conf); String transactionalRoot = (String) conf.get(Config.TRANSACTIONAL_ZOOKEEPER_ROOT); String rootDir = transactionalRoot + "/" + id + "/" + subroot; List<String> servers = (List<String>) getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, Config.STORM_ZOOKEEPER_SERVERS); Object port = getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_PORT, Config.STORM_ZOOKEEPER_PORT); ZookeeperAuthInfo auth = new ZookeeperAuthInfo(conf); CuratorFramework initter = CuratorUtils.newCuratorStarted(conf, servers, port, auth, DaemonType.WORKER.getDefaultZkAcls(conf)); _zkAcls = Utils.getWorkerACL(conf); try { TransactionalState.createNode(initter, transactionalRoot, null, null, null); } catch (KeeperException.NodeExistsException e) { } try { TransactionalState.createNode(initter, rootDir, null, _zkAcls, null); } catch (KeeperException.NodeExistsException e) { } initter.close(); _curator = CuratorUtils.newCuratorStarted(conf, servers, port, rootDir, auth, DaemonType.WORKER.getDefaultZkAcls(conf)); } catch (Exception e) { throw new RuntimeException(e); } }
@Test public void getDefaultZkAclsSecureWorkerConf() { Map<String, Object> conf = ConfigUtils.readStormConfig(); conf.put(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME, "digest"); conf.put(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD, "storm:thisisapoorpassword"); conf.put(Config.STORM_ZOOKEEPER_SUPERACL, "sasl:nimbus"); conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); conf.put(Config.NIMBUS_THRIFT_PORT, 6666); assertNull(DaemonType.UNKNOWN.getDefaultZkAcls(conf)); assertNull(DaemonType.PACEMAKER.getDefaultZkAcls(conf)); assertNull(DaemonType.SUPERVISOR.getDefaultZkAcls(conf)); assertNull(DaemonType.NIMBUS.getDefaultZkAcls(conf)); List<ACL> expected = new ArrayList<>(ZooDefs.Ids.CREATOR_ALL_ACL); expected.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", "nimbus"))); assertEquals(expected, DaemonType.WORKER.getDefaultZkAcls(conf)); } }
public CuratorFramework mkClientImpl(Map<String, Object> conf, List<String> servers, Object port, String root, final WatcherCallBack watcher, Map<String, Object> authConf, DaemonType type) { CuratorFramework fk; if (authConf != null) { fk = CuratorUtils.newCurator(conf, servers, port, root, new ZookeeperAuthInfo(authConf), type.getDefaultZkAcls(conf)); } else { fk = CuratorUtils.newCurator(conf, servers, port, root, null, type.getDefaultZkAcls(conf)); } fk.getCuratorListenable().addListener((unused, e) -> { if (e.getType().equals(CuratorEventType.WATCHED)) { WatchedEvent event = e.getWatchedEvent(); watcher.execute(event.getState(), event.getType(), event.getPath()); } }); LOG.info("Starting ZK Curator"); fk.start(); return fk; } }
@Test public void newCuratorUsesExponentialBackoffTest() throws InterruptedException { final int expectedInterval = 2400; final int expectedRetries = 10; final int expectedCeiling = 3000; Map<String, Object> config = Utils.readDefaultConfig(); config.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, expectedInterval); config.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, expectedRetries); config.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING, expectedCeiling); CuratorFramework curator = CuratorUtils.newCurator(config, Arrays.asList("bogus_server"), 42, "", DaemonType.WORKER.getDefaultZkAcls(config)); StormBoundedExponentialBackoffRetry policy = (StormBoundedExponentialBackoffRetry) curator.getZookeeperClient().getRetryPolicy(); Assert.assertEquals(policy.getBaseSleepTimeMs(), expectedInterval); Assert.assertEquals(policy.getN(), expectedRetries); Assert.assertEquals(policy.getSleepTimeMs(10, 0), expectedCeiling); }