public void startCluster() { // Start Zookeeper. _zkServer = new EmbeddedZookeeper(); _zkConnectString = "127.0.0.1:"+_zkServer.port(); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); // Start Kafka Cluster. for(int i=0;i<clusterCount;i++) { KafkaServer _kafkaServer = createKafkaServer(i,_zkConnectString); kafkaBrokerList.add(_kafkaServer); } }
void shutdown() { if (_numStarted.decrementAndGet() == 0) { log.info("Shutting down Kafka server suite"); _kafkaServer.shutdown(); _zkClient.close(); _zkServer.shutdown(); } else { log.info("Kafka server suite still in use ... not shutting down yet"); } }
public static void startServer() throws RuntimeException { if (serverStarted && serverClosed) { throw new RuntimeException("Kafka test server has already been closed. Cannot generate Kafka server twice."); } if (!serverStarted) { serverStarted = true; zkConnect = TestZKUtils.zookeeperConnect(); zkServer = new EmbeddedZookeeper(zkConnect); zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$); kafkaPort = TestUtils.choosePort(); Properties props = TestUtils.createBrokerConfig(brokerId, kafkaPort, true); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); } }
void start() throws RuntimeException { if (_numStarted.incrementAndGet() == 1) { log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort); _zkServer = new EmbeddedZookeeper(_zkConnectString); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); Properties props = kafka.utils.TestUtils.createBrokerConfig(_brokerId, _kafkaServerPort, true); props.setProperty("zookeeper.connect", _zkConnectString); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); _kafkaServer = kafka.utils.TestUtils.createServer(config, mock); } else { log.info("Kafka server suite already started... continuing"); } }
public int getZookeeperPort() { return _zkServer.port(); }
private void startSchemaRegistryServer() throws Exception { Properties props = props("listeners", "http://0.0.0.0:8081", "kafkastore.connection.url", ZK_HOST + ':' + zkServer.port()); SchemaRegistryConfig config = new SchemaRegistryConfig(props); SchemaRegistryRestApplication app = new SchemaRegistryRestApplication(config); server = app.createServer(); server.start(); }
/** * Override to set up your specific external resource. * * @throws Throwable if setup fails (which will disable {@code after} */ @Override protected void before() throws Throwable { // Start the ZK and the Broker LOG.info("init embedded Zookeeper"); zkServer = new EmbeddedZookeeper(); tmpLogDir = Files.createTempDirectory("kafka-log-dir-").toAbsolutePath(); String zkConnect = "127.0.0.1:" + zkServer.port(); LOG.info("init kafka broker"); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dir", tmpLogDir.toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKER_IP_PORT); brokerProps.setProperty("offsets.topic.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.min.isr", "1"); KafkaConfig config = new KafkaConfig(brokerProps); kafkaServer = TestUtils.createServer(config, Time.SYSTEM); kafkaServer.startup(); kafkaServer.zkClient(); adminZkClient = new AdminZkClient(kafkaServer.zkClient()); LOG.info("Creating kafka TOPIC [{}]", TOPIC); adminZkClient.createTopic(TOPIC, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$); }
void shutdown() { if (_numStarted.decrementAndGet() == 0) { log.info("Shutting down Kafka server suite"); _kafkaServer.shutdown(); _zkClient.close(); _zkServer.shutdown(); } else { log.info("Kafka server suite still in use ... not shutting down yet"); } }
EmbeddedZookeeper zkServer = new EmbeddedZookeeper(TestZKUtils.zookeeperConnect()); ZkClient zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$); Properties props = TestUtils.createBrokerConfig(0, TestUtils.choosePort(), false); KafkaConfig config = new KafkaConfig(props);
if (_numStarted.incrementAndGet() == 1) { log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort); _zkServer = new EmbeddedZookeeper(); _zkConnectString = "127.0.0.1:"+_zkServer.port(); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$);
/** * Override to tear down your specific external resource. */ @Override protected void after() { super.after(); try { FileUtils.deleteDirectory(new File(tmpLogDir.toString())); } catch (IOException e) { LOG.error("Error cleaning " + tmpLogDir.toString(), e); } if (kafkaServer != null) { kafkaServer.shutdown(); kafkaServer.awaitShutdown(); } zkServer.shutdown(); }
@Override public void afterPropertiesSet() { this.zookeeper = new EmbeddedZookeeper(); int zkConnectionTimeout = 6000; // NOSONAR magic # int zkSessionTimeout = 6000; // NOSONAR magic # this.zkConnect = "127.0.0.1:" + this.zookeeper.port(); this.zookeeperClient = new ZkClient(this.zkConnect, zkSessionTimeout, zkConnectionTimeout, ZKStringSerializer$.MODULE$);
public static void closeServer() { if (serverStarted && !serverClosed) { serverClosed = true; kafkaServer.shutdown(); zkClient.close(); zkServer.shutdown(); } }
@Before public void setUp() throws IOException, SQLException { // setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = ZkUtils.apply(zkClient, false); // setup Broker Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); kafkaServer.startup(); // create topic AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties()); pConsumer = new PhoenixConsumer(); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); conn = DriverManager.getConnection(getUrl(), props); }
@After public void cleanUp() throws Exception { kafkaServer.shutdown(); zkClient.close(); zkServer.shutdown(); conn.close(); }
private void createKafkaCluster() throws IOException { System.setProperty("zookeeper.preAllocSize", Integer.toString(128)); zkServer = new EmbeddedZookeeper(); String zkConnect = ZK_HOST + ':' + zkServer.port(); ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT, CONNECTION_TIMEOUT, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); brokerPort = getRandomPort(); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKER_HOST + ':' + brokerPort); brokerProps.setProperty("offsets.topic.replication.factor", "1"); brokerProps.setProperty("offsets.topic.num.partitions", "1"); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
this.zookeeper.shutdown();
private void createKafkaCluster() throws IOException { zkServer = new EmbeddedZookeeper(); String zkConnect = "localhost:" + zkServer.port(); ZkClient zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); KafkaConfig config = new KafkaConfig(props( "zookeeper.connect", zkConnect, "broker.id", "0", "log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString(), "offsets.topic.replication.factor", "1", "listeners", "PLAINTEXT://localhost:9092")); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
/** * Shutdown the Embedded Kafka and Zookeeper. * @throws Exception */ protected void shutdown() throws Exception { try { if(kafkaServer != null) { kafkaServer.shutdown(); } } finally { if(zkServer != null) { zkServer.shutdown(); } } }
/** * Starts the Embedded Kafka and Zookeeper Servers. * @throws Exception - If an exeption occurs during startup. */ protected void startup() throws Exception { // Setup the embedded zookeeper logger.info("Starting up Embedded Zookeeper..."); zkServer = new EmbeddedZookeeper(); zookeperConnect = ZKHOST + ":" + zkServer.port(); logger.info("Embedded Zookeeper started at: {}", zookeperConnect); // setup Broker logger.info("Starting up Embedded Kafka..."); brokerPort = Integer.toString(PortUtils.getRandomFreePort()); final Properties brokerProps = new Properties(); brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0"); brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST); brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort); brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect); brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString()); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort); }