public void setUp() throws Exception { // setup ZK zookeeper = new TestingServer(true); // setup Broker kafkaDir = new TmpPath(Files.createTempDirectory("kafka-").toAbsolutePath().toString()); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zookeeper.getConnectString()); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", kafkaDir.getPath()); brokerProps.setProperty("listeners", String.format("PLAINTEXT://%s:%d", KAFKA_HOST, KAFKA_PORT)); brokerProps.setProperty("offsets.topic.replication.factor", "1"); KafkaConfig config = new KafkaConfig(brokerProps); MockTime mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); // setup default Producer createProducer(); kafkaAdminClient = AdminClient.create(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_HOST + ":" + KAFKA_PORT)); }
/** * Override to set up your specific external resource. * * @throws Throwable if setup fails (which will disable {@code after} */ @Override protected void before() throws Throwable { // Start the ZK and the Broker LOG.info("init embedded Zookeeper"); zkServer = new EmbeddedZookeeper(); tmpLogDir = Files.createTempDirectory("kafka-log-dir-").toAbsolutePath(); String zkConnect = "127.0.0.1:" + zkServer.port(); LOG.info("init kafka broker"); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dir", tmpLogDir.toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKER_IP_PORT); brokerProps.setProperty("offsets.topic.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.min.isr", "1"); KafkaConfig config = new KafkaConfig(brokerProps); kafkaServer = TestUtils.createServer(config, Time.SYSTEM); kafkaServer.startup(); kafkaServer.zkClient(); adminZkClient = new AdminZkClient(kafkaServer.zkClient()); LOG.info("Creating kafka TOPIC [{}]", TOPIC); adminZkClient.createTopic(TOPIC, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$); }
/** * Creates and starts an embedded Kafka broker. * * @param config Broker configuration settings. Used to modify, for example, the listeners * the broker should use. Note that you cannot change some settings such as * `log.dirs`. */ KafkaEmbedded(final Properties config) throws IOException { this.tmpFolder = new TemporaryFolder(); this.tmpFolder.create(); this.logDir = tmpFolder.newFolder(); this.effectiveConfig = effectiveConfigFrom(config, logDir); final KafkaConfig kafkaConfig = new KafkaConfig(effectiveConfig, true); log.debug("Starting embedded Kafka broker (with log.dirs={} and ZK ensemble at {}) ...", logDir, zookeeperConnect()); kafka = TestUtils.createServer(kafkaConfig, new SystemTime()); log.debug("Startup of embedded Kafka broker at {} completed (with ZK ensemble at {}) ...", brokerList(), zookeeperConnect()); }
void start() throws RuntimeException { if (_numStarted.incrementAndGet() == 1) { log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort); _zkServer = new EmbeddedZookeeper(_zkConnectString); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); Properties props = kafka.utils.TestUtils.createBrokerConfig(_brokerId, _kafkaServerPort, true); props.setProperty("zookeeper.connect", _zkConnectString); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); _kafkaServer = kafka.utils.TestUtils.createServer(config, mock); } else { log.info("Kafka server suite already started... continuing"); } }
public static void startServer() throws RuntimeException { if (serverStarted && serverClosed) { throw new RuntimeException("Kafka test server has already been closed. Cannot generate Kafka server twice."); } if (!serverStarted) { serverStarted = true; zkConnect = TestZKUtils.zookeeperConnect(); zkServer = new EmbeddedZookeeper(zkConnect); zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$); kafkaPort = TestUtils.choosePort(); Properties props = TestUtils.createBrokerConfig(brokerId, kafkaPort, true); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); } }
_kafkaServer = kafka.utils.TestUtils.createServer(config, mock);
private KafkaServer createKafkaServer(int brokerId,String _zkConnectString){ int _brokerId = brokerId; int _kafkaServerPort = TestUtils.findFreePort(); Properties props = kafka.utils.TestUtils.createBrokerConfig( _brokerId, _zkConnectString, kafka.utils.TestUtils.createBrokerConfig$default$3(), kafka.utils.TestUtils.createBrokerConfig$default$4(), _kafkaServerPort, kafka.utils.TestUtils.createBrokerConfig$default$6(), kafka.utils.TestUtils.createBrokerConfig$default$7(), kafka.utils.TestUtils.createBrokerConfig$default$8(), kafka.utils.TestUtils.createBrokerConfig$default$9(), kafka.utils.TestUtils.createBrokerConfig$default$10(), kafka.utils.TestUtils.createBrokerConfig$default$11(), kafka.utils.TestUtils.createBrokerConfig$default$12(), kafka.utils.TestUtils.createBrokerConfig$default$13(), kafka.utils.TestUtils.createBrokerConfig$default$14() ); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); KafkaServer _kafkaServer = kafka.utils.TestUtils.createServer(config, mock); kafkaBrokerPortList.add(_kafkaServerPort); return _kafkaServer; }
this.brokerProperties.forEach(brokerConfigProperties::put); KafkaServer server = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), Time.SYSTEM); this.kafkaServers.add(server); if (this.kafkaPorts[i] == 0) {
@Before public void setUp() throws IOException, SQLException { // setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = ZkUtils.apply(zkClient, false); // setup Broker Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); kafkaServer.startup(); // create topic AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties()); pConsumer = new PhoenixConsumer(); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); conn = DriverManager.getConnection(getUrl(), props); }
/** * Creates and starts an embedded Kafka broker. * * @param config Broker configuration settings. Used to modify, for example, on which port the * broker should listen to. Note that you cannot change some settings such as * `log.dirs`, `port`. */ public KafkaEmbedded(Properties config) throws IOException { tmpFolder = new TemporaryFolder(); tmpFolder.create(); logDir = tmpFolder.newFolder(); effectiveConfig = effectiveConfigFrom(config); boolean loggingEnabled = true; KafkaConfig kafkaConfig = new KafkaConfig(effectiveConfig, loggingEnabled); log.debug("Starting embedded Kafka broker (with log.dirs={} and ZK ensemble at {}) ...", logDir, zookeeperConnect()); kafka = TestUtils.createServer(kafkaConfig, Time.SYSTEM); log.debug("Startup of embedded Kafka broker at {} completed (with ZK ensemble at {}) ...", brokerList(), zookeeperConnect()); }
private void createKafkaCluster() throws IOException { System.setProperty("zookeeper.preAllocSize", Integer.toString(128)); zkServer = new EmbeddedZookeeper(); String zkConnect = ZK_HOST + ':' + zkServer.port(); ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT, CONNECTION_TIMEOUT, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); brokerPort = getRandomPort(); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKER_HOST + ':' + brokerPort); brokerProps.setProperty("offsets.topic.replication.factor", "1"); brokerProps.setProperty("offsets.topic.num.partitions", "1"); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
/** * Starts the Embedded Kafka and Zookeeper Servers. * @throws Exception - If an exeption occurs during startup. */ protected void startup() throws Exception { // Setup the embedded zookeeper logger.info("Starting up Embedded Zookeeper..."); zkServer = new EmbeddedZookeeper(); zookeperConnect = ZKHOST + ":" + zkServer.port(); logger.info("Embedded Zookeeper started at: {}", zookeperConnect); // setup Broker logger.info("Starting up Embedded Kafka..."); brokerPort = Integer.toString(PortUtils.getRandomFreePort()); final Properties brokerProps = new Properties(); brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0"); brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST); brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort); brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect); brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString()); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort); }
/** * Starts the Embedded Kafka and Zookeeper Servers. * @throws Exception - If an exeption occurs during startup. */ protected void startup() throws Exception { // Setup the embedded zookeeper logger.info("Starting up Embedded Zookeeper..."); zkServer = new EmbeddedZookeeper(); zookeperConnect = ZKHOST + ":" + zkServer.port(); logger.info("Embedded Zookeeper started at: {}", zookeperConnect); // setup Broker logger.info("Starting up Embedded Kafka..."); brokerPort = Integer.toString(PortUtils.getRandomFreePort()); final Properties brokerProps = new Properties(); brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0"); brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST); brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort); brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect); brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString()); brokerProps.setProperty(KafkaConfig$.MODULE$.DeleteTopicEnableProp(), "true"); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort); }
/** * setup mini kafka and call the super to setup mini fluo */ @Before public void setupKafka() throws Exception { // Install an instance of Rya on the Accumulo cluster. installRyaInstance(); // Setup Kafka. zkServer = new EmbeddedZookeeper(); final String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); // setup Broker final Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
/** * setup mini kafka and call the super to setup mini fluo */ @Before public void setupKafka() throws Exception { // Install an instance of Rya on the Accumulo cluster. installRyaInstance(); // Setup Kafka. zkServer = new EmbeddedZookeeper(); final String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); // setup Broker final Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
private void createKafkaCluster() throws IOException { zkServer = new EmbeddedZookeeper(); String zkConnect = "localhost:" + zkServer.port(); ZkClient zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); KafkaConfig config = new KafkaConfig(props( "zookeeper.connect", zkConnect, "broker.id", "0", "log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString(), "offsets.topic.replication.factor", "1", "listeners", "PLAINTEXT://localhost:9092")); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
brokerConfigProperties.setProperty("controller.socket.timeout.ms", "1000"); brokerConfigProperties.setProperty("offsets.topic.replication.factor", "1"); this.kafkaServer = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), SystemTime$.MODULE$);
this.brokerProperties.forEach(brokerConfigProperties::put); KafkaServer server = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), Time.SYSTEM); this.kafkaServers.add(server); if (this.kafkaPorts[i] == 0) {
public EmbeddedKafkaBroker(Properties props) { this.server = TestUtils.createServer(new KafkaConfig(props), new SystemTime()); }
public void start() throws IOException { // Find port try { ServerSocket serverSocket = new ServerSocket(0); BROKERPORT = Integer.toString(serverSocket.getLocalPort()); serverSocket.close(); } catch (IOException e) { throw Throwables.propagate(e); } // Setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = BROKERHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); // Setup brokers cleanupDir(); Properties props = new Properties(); props.setProperty("zookeeper.connect", zkConnect); props.setProperty("broker.id", "0"); props.setProperty("log.dirs", KAFKA_PATH); props.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }