private List<String> getHostNames(Collection<Forest> forests) { return forests.stream().map((forest)->forest.getPreferredHost()).distinct().collect(Collectors.toList()); }
if((blackList.contains(forest.getPreferredHost()) || blackList.contains(forest.getHost())) && forest.getPreferredHostType() == HostType.REQUEST_HOST) { hostArray = new String[] {forest.getOpenReplicaHost(), forest.getAlternateHost()}; if ( blackList.size() > 0 ) { replaced = replaced.map( forest -> { if ( blackList.contains(forest.getPreferredHost()) || (forest.getPreferredHostType() == HostType.REQUEST_HOST && blackList.contains(forest.getHost()))) { if ( validHosts.length == 0 ) { if ( ! whiteList.contains(forest.getPreferredHost()) ) { if ( validHosts.length == 0 ) { throw new IllegalStateException("White list or black list rules are too restrictive:" +
/** * A utility method to return the list of hosts a Batcher should use when talking * to this database. The list is retrieved by calling getPreferredHost() on each * Forest. * * @return the list of hosts a Batcher should use */ public default String[] getPreferredHosts() { return Stream.of(listForests()).map( (forest) -> forest.getPreferredHost()).distinct().toArray(String[]::new); } }
if ( forest.getPreferredHost() == null ) { throw new IllegalStateException("Hostname must not be null for any forest"); hosts.put(forest.getPreferredHost(), forest);
@Override public synchronized QueryBatcher withForestConfig(ForestConfiguration forestConfig) { super.withForestConfig(forestConfig); Forest[] forests = forestConfig.listForests(); Set<Forest> oldForests = new HashSet<>(forestResults.keySet()); Map<String,Forest> hosts = new HashMap<>(); for ( Forest forest : forests ) { if ( forest.getPreferredHost() == null ) throw new IllegalStateException("Hostname must not be null for any forest"); hosts.put(forest.getPreferredHost(), forest); if ( forestResults.get(forest) == null ) forestResults.put(forest, new AtomicLong()); if ( forestIsDone.get(forest) == null ) forestIsDone.put(forest, new AtomicBoolean(false)); if ( retryForestMap.get(forest) == null ) retryForestMap.put(forest, new AtomicInteger(0)); } logger.info("(withForestConfig) Using forests on {} hosts for \"{}\"", hosts.keySet(), forests[0].getDatabaseName()); List<DatabaseClient> newClientList = new ArrayList<>(); for ( String host : hosts.keySet() ) { Forest forest = hosts.get(host); DatabaseClient client = getMoveMgr().getForestClient(forest); newClientList.add(client); } clientList.set(newClientList); boolean started = (threadPool != null); if ( started == true && oldForests.size() > 0 ) calucluateDeltas(oldForests, forests); return this; }
public DatabaseClient getForestClient(Forest forest) { if ( forest == null ) throw new IllegalArgumentException("forest must not be null"); if (getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) { return getPrimaryClient(); } String hostName = forest.getPreferredHost(); String key = hostName; DatabaseClient client = clientMap.get(key); if ( client != null ) return client; // since this is shared across threads, let's get an exclusive lock on it before updating it synchronized(clientMap) { // just to avoid creating unnecessary DatabaseClient instances, let's check one more time if // another thread just barely inserted an instance that matches client = clientMap.get(key); if ( client != null ) return client; client = DatabaseClientFactory.newClient( hostName, primaryClient.getPort(), forest.getDatabaseName(), primaryClient.getSecurityContext() ); clientMap.put(key, client); } return client; }
private void retry(QueryEvent queryEvent, boolean callFailListeners) { if ( isStopped() == true ) { logger.warn("Job is now stopped, aborting the retry"); return; } Forest retryForest = null; for ( Forest forest : getForestConfig().listForests() ) { if ( forest.equals(queryEvent.getForest()) ) { // while forest and queryEvent.getForest() have equivalent forest id, // we expect forest to have the currently available host info retryForest = forest; break; } } if ( retryForest == null ) { throw new IllegalStateException("Forest for queryEvent (" + queryEvent.getForest().getForestName() + ") is not in current getForestConfig()"); } // we're obviously not done with this forest forestIsDone.get(retryForest).set(false); retryForestMap.get(retryForest).incrementAndGet(); long start = queryEvent.getForestResultsSoFar() + 1; logger.trace("retryForest {} on retryHost {} at start {}", retryForest.getForestName(), retryForest.getPreferredHost(), start); QueryTask runnable = new QueryTask(getMoveMgr(), this, retryForest, query, queryEvent.getForestBatchNumber(), start, queryEvent.getJobBatchNumber(), callFailListeners); runnable.run(); } /*
@Test public void testWithWriteAndQueryBatcher() throws Exception{ if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) return; ForestConfiguration forestConfig = moveMgr.readForestConfig(); long hostNum = Stream.of(forestConfig.listForests()).map(forest->forest.getPreferredHost()).distinct().count(); if ( hostNum <= 1 ) return; // we're not in a cluster, so this test isn't valid String host1 = forestConfig.listForests()[0].getPreferredHost(); FilteredForestConfiguration ffg = new FilteredForestConfiguration(forestConfig) .withRenamedHost(host1, Inet4Address.getByName(host1).getHostAddress()); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withWhiteList(host1); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withBlackList(host1); runWithWriteAndQueryBatcher(ffg); }
assertEquals("alternatehost1", filteredForests[0].getPreferredHost()); assertEquals("openreplicahost1", filteredForests[0].getOpenReplicaHost()); assertEquals("requesthost1", filteredForests[0].getRequestHost()); assertFalse("openreplicahost2".equals(filteredForests[1].getPreferredHost())); assertEquals("host2", filteredForests[1].getHost()); assertEquals(null, filteredForests[1].getRequestHost()); assertFalse("alternatehost3".equals(filteredForests[2].getPreferredHost())); assertEquals("host3", filteredForests[2].getHost()); assertEquals(null, filteredForests[2].getOpenReplicaHost()); assertFalse("requestHost4".equals(filteredForests[3].getPreferredHost())); assertFalse("host4".equals(filteredForests[3].getHost())); assertEquals(null, filteredForests[3].getOpenReplicaHost());
assertFalse("alternatehost1".equals(filteredForests[0].getPreferredHost())); assertFalse("openreplicahost1".equals(filteredForests[0].getOpenReplicaHost())); assertFalse("requesthost1".equals(filteredForests[0].getRequestHost())); assertEquals("openreplicahost2", filteredForests[1].getPreferredHost()); assertEquals("alternatehost3", filteredForests[2].getPreferredHost()); assertEquals("requesthost4", filteredForests[3].getPreferredHost());
@Test public void testWithInvalidHosts() throws Exception{ if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) return; ForestConfiguration forestConfig = moveMgr.readForestConfig(); String host1 = forestConfig.listForests()[0].getPreferredHost(); FilteredForestConfiguration ffg = new FilteredForestConfiguration(forestConfig) .withRenamedHost("someInvalidHostName", "anotherInvalidHostName"); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withBlackList("someInvalidHostName"); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withWhiteList("someInvalidHostName") .withWhiteList(host1); runWithWriteAndQueryBatcher(ffg); }