@Override public void run() { zkUtils.close(); } };
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to check for existence * @return {@code true} if and only if the given topic exists */ public static boolean topicExists(String zkServers, String topic) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { return AdminUtils.topicExists(zkUtils, topic); } finally { zkUtils.close(); } }
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to delete, if it exists */ public static void deleteTopic(String zkServers, String topic) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { if (AdminUtils.topicExists(zkUtils, topic)) { log.info("Deleting topic {}", topic); AdminUtils.deleteTopic(zkUtils, topic); log.info("Deleted Zookeeper topic {}", topic); } else { log.info("No need to delete topic {} as it does not exist", topic); } } finally { zkUtils.close(); } }
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to create (if not already existing) * @param partitions number of topic partitions * @param topicProperties optional topic config properties */ public static void maybeCreateTopic(String zkServers, String topic, int partitions, Properties topicProperties) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { if (AdminUtils.topicExists(zkUtils, topic)) { log.info("No need to create topic {} as it already exists", topic); } else { log.info("Creating topic {} with {} partition(s)", topic, partitions); try { AdminUtils.createTopic( zkUtils, topic, partitions, 1, topicProperties, RackAwareMode.Enforced$.MODULE$); log.info("Created topic {}", topic); } catch (TopicExistsException re) { log.info("Topic {} already exists", topic); } } } finally { zkUtils.close(); } }
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param groupID consumer group to update * @param offsets mapping of (topic and) partition to offset to push to Zookeeper */ public static void setOffsets(String zkServers, String groupID, Map<Pair<String,Integer>,Long> offsets) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { offsets.forEach((topicAndPartition, offset) -> { String topic = topicAndPartition.getFirst(); int partition = topicAndPartition.getSecond(); String partitionOffsetPath = "/consumers/" + groupID + "/offsets/" + topic + "/" + partition; zkUtils.updatePersistentPath(partitionOffsetPath, Long.toString(offset), ZkUtils$.MODULE$.defaultAcls(false, "")); }); } finally { zkUtils.close(); } }
/** * @param zkUrl zookeeper connection url * @return number of brokers in this cluster */ public static int getBrokerCount(String zkUrl) { ZkUtils zkUtils = ZkUtils.apply(zkUrl, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled()); try { return zkUtils.getAllBrokersInCluster().size(); } finally { zkUtils.close(); } }
@Override public void deleteTestTopic(String topic) { ZkUtils zkUtils = getZkUtils(); try { LOG.info("Deleting topic {}", topic); ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); AdminUtils.deleteTopic(zkUtils, topic); zk.close(); } finally { zkUtils.close(); } }
@Override public void deleteTestTopic(String topic) { ZkUtils zkUtils = getZkUtils(); try { LOG.info("Deleting topic {}", topic); ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); AdminUtils.deleteTopic(zkUtils, topic); zk.close(); } finally { zkUtils.close(); } }
@Override public void deleteTestTopic(String topic) { ZkUtils zkUtils = getZkUtils(); try { LOG.info("Deleting topic {}", topic); ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); AdminUtils.deleteTopic(zkUtils, topic); zk.close(); } finally { zkUtils.close(); } }
/** * Read number of partitions for the given topic on the specified zookeeper * @param zkUrl zookeeper connection url * @param topic topic name * * @return the number of partitions of the given topic */ public static int getPartitionNumForTopic(String zkUrl, String topic) { ZkUtils zkUtils = ZkUtils.apply(zkUrl, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled()); try { Seq<String> topics = scala.collection.JavaConversions.asScalaBuffer(Arrays.asList(topic)); return zkUtils.getPartitionsForTopics(topics).apply(topic).size(); } catch (NoSuchElementException e) { return 0; } finally { zkUtils.close(); } }
zkUtils.close();
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.errorCode()); // not the first try. Sleep a bit Thread.sleep(150); } Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata(); firstPart = partitionMetadata.head(); } while (firstPart.errorCode() != 0); return firstPart.leader().get().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
private static void putOffsetInfoIntoZk(String groupId, Map<String, Map<Integer, Long>> topicOffsetsMap) { ZkUtils zkUtils = ZkUtils.apply(AuditConfig.INGESTER_ZK_CONNECT, Integer.valueOf(AuditConfig.INGESTER_ZK_SESSION_TIMEOUT_MS), Integer.valueOf(AuditConfig.INGESTER_ZK_SESSION_TIMEOUT_MS), false); try { for (Map.Entry<String, Map<Integer, Long>> topicEntry : topicOffsetsMap.entrySet()) { String zkPath = String.format("%s/%s/offsets/%s/", ZkUtils.ConsumersPath(), groupId, topicEntry.getKey()); for (Map.Entry<Integer, Long> offsetEntry : topicEntry.getValue().entrySet()) { logger.info("Put offset={} to partition={} with znode path={}", offsetEntry.getValue(), offsetEntry.getKey(), zkPath + offsetEntry.getKey()); zkUtils.updatePersistentPath(zkPath + offsetEntry.getKey(), offsetEntry.getValue().toString(), zkUtils.DefaultAcls()); } } } catch (Exception e) { logger.error("Got exception to put offset, with zkPathPrefix={}", String.format("%s/%s/offsets", ZkUtils.ConsumersPath(), groupId)); throw e; } finally { zkUtils.close(); } }
private static void removeOffsetInfoFromZk(final String groupId) { ZkUtils zkUtils = ZkUtils.apply(AuditConfig.INGESTER_ZK_CONNECT, Integer.valueOf(AuditConfig.INGESTER_ZK_SESSION_TIMEOUT_MS), Integer.valueOf(AuditConfig.INGESTER_ZK_SESSION_TIMEOUT_MS), false); try { String[] targets = new String[] {"offsets", "owners"}; for (String target : targets) { String zkPath = String.format("%s/%s/%s", ZkUtils.ConsumersPath(), groupId, target); logger.info("Remove {} with znode path={}", target, zkPath); zkUtils.deletePathRecursive(zkPath); } } catch (Exception e) { logger.error("Got exception to remove offsets or owners from zookeeper, with zkPathPrefix={}", String.format("%s/%s/", ZkUtils.ConsumersPath(), groupId)); throw e; } finally { zkUtils.close(); } }
zkUtils.close();
public static boolean createKafkaTopicIfNecessary(String brokerUri, int replFactor, int numPartitions, String topic) throws IOException { URI zkUri = URI.create(brokerUri); Preconditions.checkArgument("zk".equals(zkUri.getScheme())); String zkServerList = zkUri.getAuthority() + zkUri.getPath(); ZkUtils zkUtils = ZkUtils.apply(zkServerList, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled()); try { if (AdminUtils.topicExists(zkUtils, topic)) { return false; } try { AdminUtils.createTopic(zkUtils, topic, numPartitions, replFactor, new Properties()); } catch (TopicExistsException ignored) { return false; } catch (RuntimeException e) { throw new IOException(e); } } finally { if (zkUtils != null) { zkUtils.close(); } } return true; } }
@Override public void close() { kafkaTopicReader.shutdown(); log.debug("Kafka store reader thread shut down"); producer.close(); log.debug("Kafka store producer shut down"); zkUtils.close(); log.debug("Kafka store zookeeper client shut down"); localStore.close(); log.debug("Kafka store shut down complete"); }
public void close() { log.info("Shutting down schema registry"); jobsKafkaStore.close(); topicsKafkaStore.close(); if (masterElector != null) { masterElector.close(); } if (zkUtils != null) { zkUtils.close(); } }