private static void invokeTopicCommand(String[] args) { // jfim: Use Java security to trap System.exit in Kafka 0.9's TopicCommand System.setSecurityManager(new SecurityManager() { @Override public void checkPermission(Permission perm) { if (perm.getName().startsWith("exitVM")) { throw new SecurityException("System.exit is disabled"); } } @Override public void checkPermission(Permission perm, Object context) { checkPermission(perm); } }); try { TopicCommand.main(args); } catch (SecurityException ex) { // Do nothing, this is caused by our security manager that disables System.exit } System.setSecurityManager(null); }
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to delete, if it exists */ public static void deleteTopic(String zkServers, String topic) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { if (AdminUtils.topicExists(zkUtils, topic)) { log.info("Deleting topic {}", topic); AdminUtils.deleteTopic(zkUtils, topic); log.info("Deleted Zookeeper topic {}", topic); } else { log.info("No need to delete topic {} as it does not exist", topic); } } finally { zkUtils.close(); } }
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to check for existence * @return {@code true} if and only if the given topic exists */ public static boolean topicExists(String zkServers, String topic) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { return AdminUtils.topicExists(zkUtils, topic); } finally { zkUtils.close(); } }
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to create (if not already existing) * @param partitions number of topic partitions * @param topicProperties optional topic config properties */ public static void maybeCreateTopic(String zkServers, String topic, int partitions, Properties topicProperties) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { if (AdminUtils.topicExists(zkUtils, topic)) { log.info("No need to create topic {} as it already exists", topic); } else { log.info("Creating topic {} with {} partition(s)", topic, partitions); try { AdminUtils.createTopic( zkUtils, topic, partitions, 1, topicProperties, RackAwareMode.Enforced$.MODULE$); log.info("Created topic {}", topic); } catch (TopicExistsException re) { log.info("Topic {} already exists", topic); } } } finally { zkUtils.close(); } }
public void provisionTopic(String topic) { if (_topicConsumerMap.containsKey(topic)) { // nothing to do: return } else { // provision topic AdminUtils.createTopic(_kafkaServerSuite.getZkClient(), topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(_kafkaServerSuite.getKafkaServer()); kafka.utils.TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); KafkaConsumerSuite consumerSuite = new KafkaConsumerSuite(_kafkaServerSuite.getZkConnectString(), topic); _topicConsumerMap.put(topic, consumerSuite); } }
public static void createTopicHelper(final String topicName, final int partitions) { Properties topicProps = new Properties(); topicProps.put(TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG, "CreateTime"); topicProps.put(TopicConfig.RETENTION_MS_CONFIG, "-1"); ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false); AdminUtils.createTopic(zkUtils, topicName, partitions, 1, topicProps, RackAwareMode.Disabled$.MODULE$); org.apache.kafka.common.requests.MetadataResponse.TopicMetadata fetchTopicMetadataFromZk = AdminUtils.fetchTopicMetadataFromZk(topicName, zkUtils); logger.info("Topic Metadata: " + fetchTopicMetadataFromZk); }
public void stopClients() throws IOException { for (Map.Entry<String, KafkaConsumerSuite> consumerSuiteEntry: _topicConsumerMap.entrySet()) { consumerSuiteEntry.getValue().shutdown(); AdminUtils.deleteTopic(_kafkaServerSuite.getZkClient(), consumerSuiteEntry.getKey()); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.errorCode()); // not the first try. Sleep a bit Thread.sleep(150); } Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata(); firstPart = partitionMetadata.head(); } while (firstPart.errorCode() != 0); return firstPart.leader().get().id(); } finally { zkUtils.close(); } }
/** * Add new partitions to the Kafka topic. * * @param zkUtils ZkUtils class to use to increase replication factor. * @param topic The topic to apply the change. * @param topicMetadata Topic metadata stored in Zookeeper. * @param partitionCount The target partition count of the topic. */ private void maybeIncreaseTopicPartitionCount(ZkUtils zkUtils, String topic, MetadataResponse.TopicMetadata topicMetadata, int partitionCount) { if (partitionCount > topicMetadata.partitionMetadata().size()) { if (!ensureTopicNotUnderPartitionReassignment(zkUtils, topic)) { LOG.warn("There are ongoing partition reassignments for topic {}, skip checking its partition count.", topic); return; } AdminUtils.addPartitions(zkUtils, topic, partitionCount, "", true, RackAwareMode.Safe$.MODULE$); LOG.info("Kafka topic " + topic + " now has " + partitionCount + " partitions."); } }
private static void triggerPreferredLeaderElection(KafkaZkClient zkClient, List<PartitionInfo> partitionInfoList) { scala.collection.mutable.HashSet<TopicPartition> scalaPartitionInfoSet = new scala.collection.mutable.HashSet<>(); for (PartitionInfo javaPartitionInfo : partitionInfoList) { scalaPartitionInfoSet.add(new TopicPartition(javaPartitionInfo.topic(), javaPartitionInfo.partition())); } PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(zkClient, scalaPartitionInfoSet); }
@Before public void setUp() { super.setUp(); ZkUtils zkUtils = KafkaCruiseControlUtils.createZkUtils(zookeeper().getConnectionString()); for (int i = 0; i < NUM_TOPICS; i++) { AdminUtils.createTopic(zkUtils, "topic-" + i, NUM_PARTITIONS, 1, new Properties(), RackAwareMode.Safe$.MODULE$); } }
@AfterClass public void afterClass() { LOG.info("teardown"); if (config.manageKafkaTopic()) { // delete kafka topic AdminUtils.deleteTopic(zkUtils, TOPIC_NAME); } // remove segments if (segmentsExist) { unloadAndKillData(fullDatasourceName); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
public void provisionTopic(String topic) { if (_topicConsumerMap.containsKey(topic)) { // nothing to do: return } else { // provision topic AdminUtils.createTopic(ZkUtils.apply(_kafkaServerSuite.getZkClient(), false), topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(_kafkaServerSuite.getKafkaServer()); kafka.utils.TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); KafkaConsumerSuite consumerSuite = new KafkaConsumerSuite(_kafkaServerSuite.getZkConnectString(), topic); _topicConsumerMap.put(topic, consumerSuite); } }
@AfterClass public void afterClass() { LOG.info("teardown"); if (config.manageKafkaTopic()) { // delete kafka topic AdminUtils.deleteTopic(zkUtils, TOPIC_NAME); } // remove segments if (segmentsExist) { unloadAndKillData(fullDatasourceName); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
public void stopClients() throws IOException { for (Map.Entry<String, KafkaConsumerSuite> consumerSuiteEntry: _topicConsumerMap.entrySet()) { consumerSuiteEntry.getValue().shutdown(); AdminUtils.deleteTopic(ZkUtils.apply(_kafkaServerSuite.getZkClient(), false), consumerSuiteEntry.getKey()); } }
@Override public void deleteTestTopic(String topic) { ZkUtils zkUtils = getZkUtils(); try { LOG.info("Deleting topic {}", topic); ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); AdminUtils.deleteTopic(zkUtils, topic); zk.close(); } finally { zkUtils.close(); } }
@Override public void deleteTestTopic(String topic) { ZkUtils zkUtils = getZkUtils(); try { LOG.info("Deleting topic {}", topic); ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); AdminUtils.deleteTopic(zkUtils, topic); zk.close(); } finally { zkUtils.close(); } }
@Override public void deleteTestTopic(String topic) { ZkUtils zkUtils = getZkUtils(); try { LOG.info("Deleting topic {}", topic); ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); AdminUtils.deleteTopic(zkUtils, topic); zk.close(); } finally { zkUtils.close(); } }