/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to create (if not already existing) * @param partitions number of topic partitions * @param topicProperties optional topic config properties */ public static void maybeCreateTopic(String zkServers, String topic, int partitions, Properties topicProperties) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { if (AdminUtils.topicExists(zkUtils, topic)) { log.info("No need to create topic {} as it already exists", topic); } else { log.info("Creating topic {} with {} partition(s)", topic, partitions); try { AdminUtils.createTopic( zkUtils, topic, partitions, 1, topicProperties, RackAwareMode.Enforced$.MODULE$); log.info("Created topic {}", topic); } catch (TopicExistsException re) { log.info("Topic {} already exists", topic); } } } finally { zkUtils.close(); } }
private void ensureTopicCreated(ZkUtils zkUtils, Set<String> allTopics, String topic, long retentionMs, int replicationFactor, int partitionCount) { Properties props = new Properties(); props.setProperty(LogConfig.RetentionMsProp(), Long.toString(retentionMs)); props.setProperty(LogConfig.CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY); if (!allTopics.contains(topic)) { AdminUtils.createTopic(zkUtils, topic, partitionCount, replicationFactor, props, RackAwareMode.Safe$.MODULE$); } else { try { AdminUtils.changeTopicConfig(zkUtils, topic, props); MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topic)), zkUtils, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)).head(); maybeIncreaseTopicReplicationFactor(zkUtils, topicMetadata, replicationFactor, topic); maybeIncreaseTopicPartitionCount(zkUtils, topic, topicMetadata, partitionCount); } catch (RuntimeException re) { LOG.error("Skip updating topic " + topic + " configuration due to failure:" + re.getMessage() + "."); } } }
AdminUtils.createTopic(zkUtils, topicName, partitions, replication, topicConfig); } catch (RuntimeException e) { throw new RuntimeException(e);
AdminUtils.createTopic(zkUtils, topic, partitionCount, replicationFactor, topicConfig, RackAwareMode.Enforced$.MODULE$); } catch (TopicExistsException e) {
public void provisionTopic(String topic) { if (_topicConsumerMap.containsKey(topic)) { // nothing to do: return } else { // provision topic AdminUtils.createTopic(_kafkaServerSuite.getZkClient(), topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(_kafkaServerSuite.getKafkaServer()); kafka.utils.TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); KafkaConsumerSuite consumerSuite = new KafkaConsumerSuite(_kafkaServerSuite.getZkConnectString(), topic); _topicConsumerMap.put(topic, consumerSuite); } }
public KafkaTestBase(String topic) throws InterruptedException, RuntimeException { startServer(); this.topic = topic; AdminUtils.createTopic(zkClient, topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); Properties consumeProps = new Properties(); consumeProps.put("zookeeper.connect", zkConnect); consumeProps.put("group.id", "testConsumer"); consumeProps.put("zookeeper.session.timeout.ms", "10000"); consumeProps.put("zookeeper.sync.time.ms", "10000"); consumeProps.put("auto.commit.interval.ms", "10000"); consumeProps.put("consumer.timeout.ms", "10000"); consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(consumeProps)); Map<String, Integer> topicCountMap = new HashMap<>(); topicCountMap.put(this.topic, 1); Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap); List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(this.topic); stream = streams.get(0); iterator = stream.iterator(); }
public static void createTopicHelper(final String topicName, final int partitions) { Properties topicProps = new Properties(); topicProps.put(TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG, "CreateTime"); topicProps.put(TopicConfig.RETENTION_MS_CONFIG, "-1"); ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false); AdminUtils.createTopic(zkUtils, topicName, partitions, 1, topicProps, RackAwareMode.Disabled$.MODULE$); org.apache.kafka.common.requests.MetadataResponse.TopicMetadata fetchTopicMetadataFromZk = AdminUtils.fetchTopicMetadataFromZk(topicName, zkUtils); logger.info("Topic Metadata: " + fetchTopicMetadataFromZk); }
public void provisionTopic(String topic) { if (_topicConsumerMap.containsKey(topic)) { // nothing to do: return } else { // provision topic AdminUtils.createTopic(ZkUtils.apply(_kafkaServerSuite.getZkClient(), false), topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(_kafkaServerSuite.getKafkaServer()); kafka.utils.TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); KafkaConsumerSuite consumerSuite = new KafkaConsumerSuite(_kafkaServerSuite.getZkConnectString(), topic); _topicConsumerMap.put(topic, consumerSuite); } }
@Before public void setUp() { super.setUp(); ZkUtils zkUtils = KafkaCruiseControlUtils.createZkUtils(zookeeper().getConnectionString()); for (int i = 0; i < NUM_TOPICS; i++) { AdminUtils.createTopic(zkUtils, "topic-" + i, NUM_PARTITIONS, 1, new Properties(), RackAwareMode.Safe$.MODULE$); } }
@BeforeClass public static void initKafka() throws Exception { synchronized (TestKafkaSuit.class) { if (initCount.get() == 0) { ZookeeperTestUtil.setZookeeperSaslTestConfigProps(); System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME).getFile()); embeddedKafkaCluster = new EmbeddedKafkaCluster(); Properties topicProps = new Properties(); zkClient = new ZkClient(embeddedKafkaCluster.getZkServer().getConnectionString(), SESSION_TIMEOUT, CONN_TIMEOUT, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false); AdminUtils.createTopic(zkUtils, TestQueryConstants.JSON_TOPIC, 1, 1, topicProps, RackAwareMode.Disabled$.MODULE$); org.apache.kafka.common.requests.MetadataResponse.TopicMetadata fetchTopicMetadataFromZk = AdminUtils .fetchTopicMetadataFromZk(TestQueryConstants.JSON_TOPIC, zkUtils); logger.info("Topic Metadata: " + fetchTopicMetadataFromZk); KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(), StringSerializer.class); generator.populateJsonMsgIntoKafka(TestQueryConstants.JSON_TOPIC, NUM_JSON_MSG); } initCount.incrementAndGet(); runningSuite = true; } logger.info("Initialized Embedded Zookeeper and Kafka"); }
Properties topicConfig = new Properties(); AdminUtils.createTopic( zkUtils, TOPIC_NAME,
int replicationFactor = 1; Properties topicConfig = new Properties(); AdminUtils.createTopic( zkUtils, TOPIC_NAME,
public static boolean createKafkaTopicIfNecessary(String brokerUri, int replFactor, int numPartitions, String topic) throws IOException { URI zkUri = URI.create(brokerUri); Preconditions.checkArgument("zk".equals(zkUri.getScheme())); String zkServerList = zkUri.getAuthority() + zkUri.getPath(); ZkUtils zkUtils = ZkUtils.apply(zkServerList, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled()); try { if (AdminUtils.topicExists(zkUtils, topic)) { return false; } try { AdminUtils.createTopic(zkUtils, topic, numPartitions, replFactor, new Properties()); } catch (TopicExistsException ignored) { return false; } catch (RuntimeException e) { throw new IOException(e); } } finally { if (zkUtils != null) { zkUtils.close(); } } return true; } }
@Before public void setUp() throws IOException, SQLException { // setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = ZkUtils.apply(zkClient, false); // setup Broker Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); kafkaServer.startup(); // create topic AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties()); pConsumer = new PhoenixConsumer(); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); conn = DriverManager.getConnection(getUrl(), props); }
/** * Create a Kafka topic with the given parameters. * * @param topic The name of the topic. * @param partitions The number of partitions for this topic. * @param replication The replication factor for (partitions of) this topic. * @param topicConfig Additional topic-level configuration settings. */ public void createTopic(String topic, int partitions, int replication, Properties topicConfig) { log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }", topic, partitions, replication, topicConfig); // Note: You must initialize the ZkClient with ZKStringSerializer. If you don't, then // createTopic() will only seem to work (it will return without error). The topic will exist in // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the // topic. ZkClient zkClient = new ZkClient( zookeeperConnect(), DEFAULT_ZK_SESSION_TIMEOUT_MS, DEFAULT_ZK_CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$); boolean isSecure = false; ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure); AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$); zkClient.close(); }
@VisibleForTesting protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) { int numPartitions = atlasProperties.getInt("atlas.notification.hook.numthreads", 1); int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1); AdminUtils.createTopic(zkUtils, topicName, numPartitions, numReplicas, new Properties(), RackAwareMode.Enforced$.MODULE$); LOG.warn("Created topic {} with partitions {} and replicas {}", topicName, numPartitions, numReplicas); }
@VisibleForTesting protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) { int numPartitions = atlasProperties.getInt("atlas.notification.hook.numthreads", 1); int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1); AdminUtils.createTopic(zkUtils, topicName, numPartitions, numReplicas, new Properties(), RackAwareMode.Enforced$.MODULE$); LOG.warn("Created topic {} with partitions {} and replicas {}", topicName, numPartitions, numReplicas); }
public void createTopics(String... topics) { for (String topic : topics) { AdminUtils.createTopic(getZkUtils(), topic, 2, 1, new Properties(), null); } }
public void createTopics(String... topics) { for (String topic : topics) { AdminUtils.createTopic(getZkUtils(), topic, 2, 1, new Properties(), null); } }
public String createNewTopic(String newTopic, int partitions) { ZkUtils zkUtils = new ZkUtils(embeddedKafka.zkClient(), null, false); Properties props = new Properties(); AdminUtils.createTopic(zkUtils, newTopic, partitions, 1, props, null); waitForTopic(newTopic, partitions, true); return newTopic; }