@Override public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception { RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper = ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore"); ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore( "/workers", stateStorageHelper); ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]); ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0); return new ZooKeeperMesosWorkerStore( zooKeeperStateHandleStore, frameworkId, totalTaskCount); }
@Override public CheckpointIDCounter createCheckpointIDCounter(JobID jobID) throws Exception { return ZooKeeperUtils.createCheckpointIDCounter(client, config, jobID); }
@Override public CompletedCheckpointStore createCheckpointStore(JobID jobId, int maxNumberOfCheckpointsToRetain, ClassLoader userClassLoader) throws Exception { return ZooKeeperUtils.createCompletedCheckpoints(client, config, jobId, maxNumberOfCheckpointsToRetain, executor); }
/** * Creates a {@link ZooKeeperLeaderElectionService} instance. * * @param client The {@link CuratorFramework} ZooKeeper client to use * @param configuration {@link Configuration} object containing the configuration values * @return {@link ZooKeeperLeaderElectionService} instance. */ public static ZooKeeperLeaderElectionService createLeaderElectionService( CuratorFramework client, Configuration configuration) throws Exception { return createLeaderElectionService(client, configuration, ""); }
@Override public LeaderRetrievalService getResourceManagerLeaderRetriever() { return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH); }
public ZooKeeperUtilityFactory(Configuration configuration, String path) throws Exception { Preconditions.checkNotNull(path, "path"); root = ZooKeeperUtils.startCuratorFramework(configuration); root.newNamespaceAwareEnsurePath(path).ensure(root.getZookeeperClient()); facade = root.usingNamespace(ZooKeeperUtils.generateZookeeperPath(root.getNamespace(), path)); }
public static HighAvailabilityServices createAvailableOrEmbeddedServices( Configuration config, Executor executor) throws Exception { HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config); switch (highAvailabilityMode) { case NONE: return new EmbeddedHaServices(executor); case FILESYSTEM: throw new UnsupportedOperationException("to be implemented"); case ZOOKEEPER: BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config); return new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(config), executor, config, blobStoreService); default: throw new Exception("High availability mode " + highAvailabilityMode + " is not supported."); } }
/** * Creates a {@link ZooKeeperStateHandleStore} instance with the provided arguments. * * @param zkStateHandleStorePath specifying the path in ZooKeeper to store the state handles to * @param stateStorageHelper storing the actual state data * @param <T> Type of the state to be stored * @return a ZooKeeperStateHandleStore instance * @throws Exception if ZooKeeper could not create the provided state handle store path in * ZooKeeper */ public <T extends Serializable> ZooKeeperStateHandleStore<T> createZooKeeperStateHandleStore( String zkStateHandleStorePath, RetrievableStateStorageHelper<T> stateStorageHelper) throws Exception { facade.newNamespaceAwareEnsurePath(zkStateHandleStorePath).ensure(facade.getZookeeperClient()); CuratorFramework stateHandleStoreFacade = facade.usingNamespace( ZooKeeperUtils.generateZookeeperPath( facade.getNamespace(), zkStateHandleStorePath)); return new ZooKeeperStateHandleStore<>(stateHandleStoreFacade, stateStorageHelper); }
@Override public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception { return ZooKeeperUtils.createSubmittedJobGraphs(client, configuration); }
/** * Returns the port range for the common {@link RpcService}. * * @param configuration to extract the port range from * @return Port range for the common {@link RpcService} */ protected String getRPCPortRange(Configuration configuration) { if (ZooKeeperUtils.isZooKeeperRecoveryMode(configuration)) { return configuration.getString(HighAvailabilityOptions.HA_JOB_MANAGER_PORT_RANGE); } else { return String.valueOf(configuration.getInteger(JobManagerOptions.PORT)); } }
/** * Creates a {@link ZooKeeperLeaderElectionService} instance. * * @param client The {@link CuratorFramework} ZooKeeper client to use * @param configuration {@link Configuration} object containing the configuration values * @return {@link ZooKeeperLeaderElectionService} instance. */ public static ZooKeeperLeaderElectionService createLeaderElectionService( CuratorFramework client, Configuration configuration) throws Exception { return createLeaderElectionService(client, configuration, ""); }
@Override public LeaderRetrievalService getDispatcherLeaderRetriever() { return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH); }
public ZooKeeperUtilityFactory(Configuration configuration, String path) throws Exception { Preconditions.checkNotNull(path, "path"); root = ZooKeeperUtils.startCuratorFramework(configuration); root.newNamespaceAwareEnsurePath(path).ensure(root.getZookeeperClient()); facade = root.usingNamespace(ZooKeeperUtils.generateZookeeperPath(root.getNamespace(), path)); }
public static HighAvailabilityServices createAvailableOrEmbeddedServices( Configuration config, Executor executor) throws Exception { HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config); switch (highAvailabilityMode) { case NONE: return new EmbeddedHaServices(executor); case ZOOKEEPER: BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config); return new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(config), executor, config, blobStoreService); default: throw new Exception("High availability mode " + highAvailabilityMode + " is not supported."); } }
/** * Creates a {@link ZooKeeperStateHandleStore} instance with the provided arguments. * * @param zkStateHandleStorePath specifying the path in ZooKeeper to store the state handles to * @param stateStorageHelper storing the actual state data * @param <T> Type of the state to be stored * @return a ZooKeeperStateHandleStore instance * @throws Exception if ZooKeeper could not create the provided state handle store path in * ZooKeeper */ public <T extends Serializable> ZooKeeperStateHandleStore<T> createZooKeeperStateHandleStore( String zkStateHandleStorePath, RetrievableStateStorageHelper<T> stateStorageHelper) throws Exception { facade.newNamespaceAwareEnsurePath(zkStateHandleStorePath).ensure(facade.getZookeeperClient()); CuratorFramework stateHandleStoreFacade = facade.usingNamespace( ZooKeeperUtils.generateZookeeperPath( facade.getNamespace(), zkStateHandleStorePath)); return new ZooKeeperStateHandleStore<>(stateHandleStoreFacade, stateStorageHelper); }
@Override public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception { return ZooKeeperUtils.createSubmittedJobGraphs(client, configuration); }
/** * Returns the port range for the common {@link RpcService}. * * @param configuration to extract the port range from * @return Port range for the common {@link RpcService} */ protected String getRPCPortRange(Configuration configuration) { if (ZooKeeperUtils.isZooKeeperRecoveryMode(configuration)) { return configuration.getString(HighAvailabilityOptions.HA_JOB_MANAGER_PORT_RANGE); } else { return String.valueOf(configuration.getInteger(JobManagerOptions.PORT)); } }
/** * Creates a {@link ZooKeeperLeaderElectionService} instance. * * @param client The {@link CuratorFramework} ZooKeeper client to use * @param configuration {@link Configuration} object containing the configuration values * @return {@link ZooKeeperLeaderElectionService} instance. */ public static ZooKeeperLeaderElectionService createLeaderElectionService( CuratorFramework client, Configuration configuration) throws Exception { return createLeaderElectionService(client, configuration, ""); }
@Override public LeaderRetrievalService getWebMonitorLeaderRetriever() { return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH); }
public ZooKeeperUtilityFactory(Configuration configuration, String path) throws Exception { Preconditions.checkNotNull(path, "path"); root = ZooKeeperUtils.startCuratorFramework(configuration); root.newNamespaceAwareEnsurePath(path).ensure(root.getZookeeperClient()); facade = root.usingNamespace(ZooKeeperUtils.generateZookeeperPath(root.getNamespace(), path)); }