private void startKafka() { _zookeeperInstance = ZkStarter.startLocalZkServer(); _kafkaStarter = KafkaStarterUtils .startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic("airlineStatsEvents", KafkaStarterUtils.DEFAULT_ZK_STR, 10); }
public static KafkaServerStartable startServer(final int port, final int brokerId, final String zkStr, final String logDirPath, final Properties configuration) { // Create the ZK nodes for Kafka, if needed int indexOfFirstSlash = zkStr.indexOf('/'); if (indexOfFirstSlash != -1) { String bareZkUrl = zkStr.substring(0, indexOfFirstSlash); String zkNodePath = zkStr.substring(indexOfFirstSlash); ZkClient client = new ZkClient(bareZkUrl); client.createPersistent(zkNodePath, true); client.close(); } File logDir = new File(logDirPath); logDir.mkdirs(); configureKafkaPort(configuration, port); configureZkConnectionString(configuration, zkStr); configureBrokerId(configuration, brokerId); configureKafkaLogDirectory(configuration, logDir); configuration.put("zookeeper.session.timeout.ms", "60000"); KafkaConfig config = new KafkaConfig(configuration); KafkaServerStartable serverStartable = new KafkaServerStartable(config); serverStartable.startup(); return serverStartable; }
public static Properties getDefaultKafkaConfiguration() { final Properties configuration = new Properties(); // Enable topic deletion by default for integration tests configureTopicDeletion(configuration, true); // Set host name configureHostName(configuration, "localhost"); return configuration; }
@Override public boolean execute() throws IOException { KafkaStarterUtils.startServer(_port, _brokerId, _zkAddress, KafkaStarterUtils.getDefaultKafkaConfiguration()); LOGGER.info("Start kafka at localhost:" + _port + " in thread " + Thread.currentThread().getName()); savePID(System.getProperty("java.io.tmpdir") + File.separator + ".kafka.pid"); return true; } }
protected void startKafka() { _kafkaStarters = KafkaStarterUtils .startServers(getNumKafkaBrokers(), KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic(getKafkaTopic(), KafkaStarterUtils.DEFAULT_ZK_STR, getNumKafkaPartitions()); }
public static List<KafkaServerStartable> startServers(final int brokerCount, final int port, final String zkStr, final Properties configuration) { List<KafkaServerStartable> startables = new ArrayList<>(brokerCount); for (int i = 0; i < brokerCount; i++) { startables.add(startServer(port + i, i, zkStr, "/tmp/kafka-" + Double.toHexString(Math.random()), configuration)); } return startables; }
@Override public void run() { try { printStatus(Color.GREEN, "***** Shutting down hybrid quick start *****"); stream.shutdown(); Thread.sleep(2000); runner.stop(); KafkaStarterUtils.stopServer(_kafkaStarter); ZkStarter.stopLocalZkServer(_zookeeperInstance); FileUtils.deleteDirectory(_offlineQuickStartDataDir); FileUtils.deleteDirectory(_realtimeQuickStartDataDir); } catch (Exception e) { e.printStackTrace(); } } });
public static KafkaServerStartable startServer(final int port, final int brokerId, final String zkStr, final Properties configuration) { return startServer(port, brokerId, zkStr, "/tmp/kafka-" + Double.toHexString(Math.random()), configuration); }
@Override public void run() { try { printStatus(Color.GREEN, "***** Shutting down realtime quick start *****"); meetupRSVPProvider.stopPublishing(); runner.stop(); KafkaStarterUtils.stopServer(kafkaStarter); ZkStarter.stopLocalZkServer(zookeeperInstance); FileUtils.deleteDirectory(quickStartDataDir); } catch (Exception e) { e.printStackTrace(); } } });
.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic(getKafkaTopic(), KafkaStarterUtils.DEFAULT_ZK_STR, 10);
@AfterClass public void tearDown() throws Exception { dropRealtimeTable(getTableName()); stopServer(); stopBroker(); stopController(); for (KafkaServerStartable kafkaStarter : _kafkaStarters) { KafkaStarterUtils.stopServer(kafkaStarter); } stopZk(); FileUtils.deleteDirectory(_tempDir); }
.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic(getKafkaTopic(), KafkaStarterUtils.DEFAULT_ZK_STR, 10);
@AfterClass public void tearDown() throws Exception { if (!_enabled) { return; } String tableName = getTableName(); dropOfflineTable(tableName); dropRealtimeTable(tableName); stopServer(); stopBroker(); stopController(); KafkaStarterUtils.stopServer(_kafkaStarter); stopZk(); FileUtils.deleteDirectory(_tempDir); } }
final ZkStarter.ZookeeperInstance zookeeperInstance = ZkStarter.startLocalZkServer(); final KafkaServerStartable kafkaStarter = KafkaStarterUtils .startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic("meetupRSVPEvents", KafkaStarterUtils.DEFAULT_ZK_STR, 10); printStatus(Color.CYAN, "***** Starting Zookeeper, controller, server and broker *****"); runner.startAll();
protected void startHybridCluster() throws Exception { // Start Zk and Kafka startZk(); _kafkaStarter = KafkaStarterUtils .startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); // Create Kafka topic KafkaStarterUtils.createTopic(getKafkaTopic(), KafkaStarterUtils.DEFAULT_ZK_STR, getNumKafkaPartitions()); // Start the Pinot cluster ControllerConf config = getDefaultControllerConfiguration(); config.setTenantIsolationEnabled(false); startController(config); startBroker(); startServers(2); // Create tenants createBrokerTenant(TENANT_NAME, 1); createServerTenant(TENANT_NAME, 1, 1); }
_kafkaStarter = KafkaStarterUtils.startServer(KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KAFKA_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic(getKafkaTopic(), KAFKA_ZK_STR, getNumKafkaPartitions());