static WordCountInteractiveQueriesRestService startRestProxy(final KafkaStreams streams, final int port, final String host) throws Exception { final HostInfo hostInfo = new HostInfo(host, port); final WordCountInteractiveQueriesRestService wordCountInteractiveQueriesRestService = new WordCountInteractiveQueriesRestService(streams, hostInfo); wordCountInteractiveQueriesRestService.start(port); return wordCountInteractiveQueriesRestService; }
@RequestMapping("/charts/top-five") @SuppressWarnings("unchecked") public List<SongPlayCountBean> topFive(@RequestParam(value="genre") String genre) { HostInfo hostInfo = interactiveQueryService.getHostInfo(KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE, KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, new StringSerializer()); if (interactiveQueryService.getCurrentHostInfo().equals(hostInfo)) { logger.info("Top Five songs request served from same host: " + hostInfo); return topFiveSongs(KafkaStreamsInteractiveQuerySample.TOP_FIVE_KEY, KafkaStreamsInteractiveQuerySample.TOP_FIVE_SONGS_STORE); } else { //find the store from the proper instance. logger.info("Top Five songs request served from different host: " + hostInfo); RestTemplate restTemplate = new RestTemplate(); return restTemplate.postForObject( String.format("http://%s:%d/%s", hostInfo.host(), hostInfo.port(), "charts/top-five?genre=Punk"), "punk", List.class); } }
public String host() { return hostInfo.host(); } public int port() {
public int port() { return hostInfo.port(); }
@Override public boolean equals(final Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; final StreamsMetadata that = (StreamsMetadata) o; if (!hostInfo.equals(that.hostInfo)) return false; if (!stateStoreNames.equals(that.stateStoreNames)) return false; return topicPartitions.equals(that.topicPartitions); }
@Override public int hashCode() { int result = hostInfo.hashCode(); result = 31 * result + stateStoreNames.hashCode(); result = 31 * result + topicPartitions.hashCode(); return result; }
songPlayCount.getSongId(), new LongSerializer()); if (interactiveQueryService.getCurrentHostInfo().equals(hostInfo)) { logger.info("Song info request served from same host: " + hostInfo); RestTemplate restTemplate = new RestTemplate(); SongBean song = restTemplate.postForObject( String.format("http://%s:%d/%s", hostInfo.host(), hostInfo.port(), "song/idx?id=" + songPlayCount.getSongId()), "id", SongBean.class); results.add(new SongPlayCountBean(song.getArtist(),song.getAlbum(), song.getName(), songPlayCount.getPlays()));
private void rebuildMetadata(final Map<HostInfo, Set<TopicPartition>> currentState) { allMetadata.clear(); if (currentState.isEmpty()) { return; } final Map<String, List<String>> stores = builder.stateStoreNameToSourceTopics(); for (final Map.Entry<HostInfo, Set<TopicPartition>> entry : currentState.entrySet()) { final HostInfo key = entry.getKey(); final Set<TopicPartition> partitionsForHost = new HashSet<>(entry.getValue()); final Set<String> storesOnHost = new HashSet<>(); for (final Map.Entry<String, List<String>> storeTopicEntry : stores.entrySet()) { final List<String> topicsForStore = storeTopicEntry.getValue(); if (hasPartitionsForAnyTopics(topicsForStore, partitionsForHost)) { storesOnHost.add(storeTopicEntry.getKey()); } } storesOnHost.addAll(globalStores); final StreamsMetadata metadata = new StreamsMetadata(key, storesOnHost, partitionsForHost); allMetadata.add(metadata); if (key.equals(thisHost)) { myMetadata = metadata; } } }
final String schemaRegistryUrl = args.length > 2 ? args[2] : "http://localhost:8081"; final String restEndpointHostname = args.length > 3 ? args[3] : DEFAULT_REST_ENDPOINT_HOSTNAME; final HostInfo restEndpoint = new HostInfo(restEndpointHostname, restEndpointPort);
/** * Start an embedded Jetty Server * @throws Exception */ void start() throws Exception { ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); context.setContextPath("/"); jettyServer = new Server(); jettyServer.setHandler(context); ResourceConfig rc = new ResourceConfig(); rc.register(this); rc.register(JacksonFeature.class); ServletContainer sc = new ServletContainer(rc); ServletHolder holder = new ServletHolder(sc); context.addServlet(holder, "/*"); final ServerConnector connector = new ServerConnector(jettyServer); connector.setHost(hostInfo.host()); connector.setPort(hostInfo.port()); jettyServer.addConnector(connector); context.start(); try { jettyServer.start(); } catch (final java.net.SocketException exception) { log.error("Unavailable: " + hostInfo.host() + ":" + hostInfo.port()); throw new Exception(exception.toString()); } }
private void createStreams(final String host) throws Exception { appServerPort = randomFreeLocalPort(); streams = KafkaMusicExample.createChartsStreams(CLUSTER.bootstrapServers(), CLUSTER.schemaRegistryUrl(), appServerPort, TestUtils.tempDirectory().getPath(), host); int count = 0; final int maxTries = 3; while (count <= maxTries) { try { // Starts the Rest Service on the provided host:port restProxy = KafkaMusicExample.startRestProxy(streams, new HostInfo(host, appServerPort)); } catch (final Exception ex) { log.error("Could not start Rest Service due to: " + ex.toString()); } count++; } }
private static void decodeGlobalAssignmentData(final AssignmentInfo assignmentInfo, final DataInputStream in) throws IOException { assignmentInfo.partitionsByHost = new HashMap<>(); final int numEntries = in.readInt(); for (int i = 0; i < numEntries; i++) { final HostInfo hostInfo = new HostInfo(in.readUTF(), in.readInt()); assignmentInfo.partitionsByHost.put(hostInfo, readTopicPartitions(in)); } }
private void encodePartitionsByHost(final DataOutputStream out) throws IOException { // encode partitions by host out.writeInt(partitionsByHost.size()); for (final Map.Entry<HostInfo, Set<TopicPartition>> entry : partitionsByHost.entrySet()) { final HostInfo hostInfo = entry.getKey(); out.writeUTF(hostInfo.host()); out.writeInt(hostInfo.port()); writeTopicPartitions(out, entry.getValue()); } }
private static HostInfo parseHostInfo(final String endPoint) { if (endPoint == null || endPoint.trim().isEmpty()) { return StreamsMetadataState.UNKNOWN_HOST; } final String host = getHost(endPoint); final Integer port = getPort(endPoint); if (host == null || port == null) { throw new ConfigException(String.format("Error parsing host address %s. Expected format host:port.", endPoint)); } return new HostInfo(host, port); }
/** * Gets the current {@link HostInfo} that the calling kafka streams application is running on. * * Note that the end user applications must provide `applicaiton.server` as a configuration property * when calling this method. If this is not available, then null is returned. * * @return the current {@link HostInfo} */ public HostInfo getCurrentHostInfo() { Map<String, String> configuration = this.binderConfigurationProperties.getConfiguration(); if (configuration.containsKey("application.server")) { String applicationServer = configuration.get("application.server"); String[] splits = StringUtils.split(applicationServer, ":"); return new HostInfo(splits[0], Integer.valueOf(splits[1])); } return null; }
ClientMetadata(final String endPoint) { // get the host info if possible if (endPoint != null) { final String host = getHost(endPoint); final Integer port = getPort(endPoint); if (host == null || port == null) { throw new ConfigException(String.format("Error parsing host address %s. Expected format host:port.", endPoint)); } hostInfo = new HostInfo(host, port); } else { hostInfo = null; } // initialize the consumer memberIds consumers = new HashSet<>(); // initialize the client state state = new ClientState(); }