public void updateExecutor(final Executor executor) throws ExecutorManagerException { final String UPDATE = "UPDATE executors SET host=?, port=?, active=? where id=?"; try { final int rows = this.dbOperator.update(UPDATE, executor.getHost(), executor.getPort(), executor.isActive(), executor.getId()); if (rows == 0) { throw new ExecutorManagerException("No executor with id :" + executor.getId()); } } catch (final SQLException e) { throw new ExecutorManagerException("Error inactivating executor " + executor.getId(), e); } }
@Override public List<Executor> handle(final ResultSet rs) throws SQLException { if (!rs.next()) { return Collections.emptyList(); } final List<Executor> executors = new ArrayList<>(); do { final int id = rs.getInt(1); final String host = rs.getString(2); final int port = rs.getInt(3); final boolean active = rs.getBoolean(4); final Executor executor = new Executor(id, host, port, active); executors.add(executor); } while (rs.next()); return executors; } }
@Test public void testExecutorActivation() throws Exception { final Executor executor = this.executorDao.addExecutor("localhost1", 12345); assertThat(executor.isActive()).isFalse(); executor.setActive(true); this.executorDao.updateExecutor(executor); final Executor fetchedExecutor = this.executorDao.fetchExecutor(executor.getId()); assertThat(fetchedExecutor.isActive()).isTrue(); } }
/** * <pre> * function to register the static remaining flow size filter. * NOTE : this is a static filter which means the filter will be filtering based on the system * standard which is not * Coming for the passed flow. * Ideally this filter will make sure only the executor hasn't reached the Max allowed # * of * executing flows. * </pre> */ private static FactorFilter<Executor, ExecutableFlow> getStaticRemainingFlowSizeFilter() { return FactorFilter .create(STATICREMAININGFLOWSIZE_FILTER_NAME, (filteringTarget, referencingObject) -> { if (null == filteringTarget) { logger.debug(String.format("%s : filtering out the target as it is null.", STATICREMAININGFLOWSIZE_FILTER_NAME)); return false; } final ExecutorInfo stats = filteringTarget.getExecutorInfo(); if (null == stats) { logger.debug(String.format("%s : filtering out %s as it's stats is unavailable.", STATICREMAININGFLOWSIZE_FILTER_NAME, filteringTarget.toString())); return false; } return stats.getRemainingFlowCapacity() > 0; }); }
private Map<Integer, Executor> getActiveExecutors() { final Map<Integer, Executor> executorMap = new HashMap<>(); try { final List<Executor> executors = this.executorLoader.fetchActiveExecutors(); for (final Executor executor : executors) { executorMap.put(executor.getId(), executor); } } catch (final ExecutorManagerException e) { log.error("Fetching executors failed!", e); } return executorMap; }
.callWithExecutionId(executor.getHost(), executor.getPort(), ConnectorParams.PING_ACTION, null, null); if (results == null || results.containsKey(ConnectorParams.RESPONSE_ERROR) || !results .containsKey(ConnectorParams.STATUS_PARAM) || !results.get(ConnectorParams.STATUS_PARAM) .equals(ConnectorParams.RESPONSE_ALIVE)) { throw new ExecutorManagerException("Status of executor " + executor.getId() + " is " + "not alive."); } else { if (this.executorFailureCount.containsKey(executor.getId())) { this.executorFailureCount.put(executor.getId(), 0);
() -> this.apiGateway.callForJsonType(executor.getHost(), executor.getPort(), "/serverStatistics", null, ExecutorInfo.class)); futures.add(new Pair<>(executor, fetchExecutionInfo)); for (final Pair<Executor, Future<ExecutorInfo>> refreshPair : futures) { final Executor executor = refreshPair.getFirst(); executor.setExecutorInfo(null); // invalidate cached ExecutorInfo try { executor.setExecutorInfo(executorInfo); logger.info(String.format( "Successfully refreshed executor: %s with executor info : %s",
private void handleException(final Entry<Optional<Executor>, List<ExecutableFlow>> entry, final Executor executor, final ExecutorManagerException e, final ArrayList<ExecutableFlow> finalizeFlows) { logger.error("Failed to get update from executor " + executor.getHost(), e); boolean sendUnresponsiveEmail = false; final boolean executorRemoved = isExecutorRemoved(executor.getId()); for (final ExecutableFlow flow : entry.getValue()) { final Pair<ExecutionReference, ExecutableFlow> pair =
@Test public void testCreatingExecutorSelectorWithEmptyFilterComparatorList() throws Exception { final List<Executor> executorList = new ArrayList<>(); executorList.add(new Executor(1, "host1", 80, true)); executorList.add(new Executor(2, "host2", 80, true)); executorList.add(new Executor(3, "host3", 80, true)); executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 89, 0)); executorList.get(1) .setExecutorInfo(new ExecutorInfo(50, 14095, 50, System.currentTimeMillis(), 90, 0)); executorList.get(2) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 90, 0)); final ExecutableFlow flow = new ExecutableFlow(); final ExecutorSelector selector = new ExecutorSelector(null, null); final Executor executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(2), executor); }
@Test public void testFetchActiveExecutors() throws Exception { final List<Executor> executors = addTestExecutors(); executors.get(0).setActive(true); this.executorDao.updateExecutor(executors.get(0)); final List<Executor> fetchedExecutors = this.executorDao.fetchActiveExecutors(); assertThat(executors.size()).isEqualTo(fetchedExecutors.size() + 2); assertThat(executors.get(0)).isEqualTo(fetchedExecutors.get(0)); }
@Override public int compare(final Executor o1, final Executor o2) { final ExecutorInfo stat1 = o1.getExecutorInfo(); final ExecutorInfo stat2 = o2.getExecutorInfo(); final Integer result = 0; if (statisticsObjectCheck(stat1, stat2, NUMOFASSIGNEDFLOW_COMPARATOR_NAME)) { return result; } return ((Integer) stat1.getRemainingFlowCapacity()) .compareTo(stat2.getRemainingFlowCapacity()); } });
@Override public List<Executor> fetchActiveExecutors() throws ExecutorManagerException { final List<Executor> activeExecutors = new ArrayList<>(); for (final Executor executor : this.executors) { if (executor.isActive()) { activeExecutors.add(executor); } } return activeExecutors; }
@Override public Set<String> getPrimaryServerHosts() { final HashSet<String> ports = new HashSet<>(); try { for (final Executor executor : this.executorLoader.fetchActiveExecutors()) { ports.add(executor.getHost() + ":" + executor.getPort()); } } catch (final ExecutorManagerException e) { logger.error("Failed to get primary server hosts.", e); } return ports; }
@Override public Executor fetchExecutor(final int executorId) throws ExecutorManagerException { for (final Executor executor : this.executors) { if (executor.getId() == executorId) { return executor; } } return null; }
() -> this.apiGateway.callForJsonType(executor.getHost(), executor.getPort(), "/serverStatistics", null, ExecutorInfo.class)); futures.add(new Pair<>(executor, fetchExecutionInfo)); for (final Pair<Executor, Future<ExecutorInfo>> refreshPair : futures) { final Executor executor = refreshPair.getFirst(); executor.setExecutorInfo(null); // invalidate cached ExecutorInfo try { executor.setExecutorInfo(executorInfo); logger.info(String.format( "Successfully refreshed executor: %s with executor info : %s",
@Test public void testExecutorSelectorE2E() throws Exception { final List<String> filterList = new ArrayList<>(ExecutorFilter.getAvailableFilterNames()); final Map<String, Integer> comparatorMap; comparatorMap = new HashMap<>(); final List<Executor> executorList = new ArrayList<>(); executorList.add(new Executor(1, "host1", 80, true)); executorList.add(new Executor(2, "host2", 80, true)); executorList.add(new Executor(3, "host3", 80, true)); executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 89, 0)); executorList.get(1) .setExecutorInfo(new ExecutorInfo(50, 14095, 50, System.currentTimeMillis(), 90, 0)); executorList.get(2) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 90, 0)); final ExecutableFlow flow = new ExecutableFlow(); for (final String name : ExecutorComparator.getAvailableComparatorNames()) { comparatorMap.put(name, 1); } final ExecutorSelector selector = new ExecutorSelector(filterList, comparatorMap); Executor executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(0), executor); // simulate that once the flow is assigned, executor1's remaining TMP storage dropped to 2048 // now we do the getBest again executor3 is expected to be selected as it has a earlier last dispatched time. executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 4095, 50, System.currentTimeMillis(), 90, 1)); executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(2), executor); }