public boolean hasResourcesAvailable() { return !isDead && getNumberOfAvailableSlots() > 0; }
/** * Unregisters all currently registered TaskManagers from the InstanceManager. */ public void unregisterAllTaskManagers() { for(Instance instance: registeredHostsById.values()) { deadHosts.add(instance.getTaskManagerID()); instance.markDead(); totalNumberOfAliveTaskSlots -= instance.getTotalNumberOfSlots(); notifyDeadInstance(instance); } registeredHostsById.clear(); registeredHostsByResource.clear(); }
private void removeInstance(Instance instance) { if (instance == null) { throw new NullPointerException(); } allInstances.remove(instance); instancesWithAvailableResources.remove(instance.getTaskManagerID()); String instanceHostName = instance.getTaskManagerLocation().getHostname(); Set<Instance> instanceSet = allInstancesByHost.get(instanceHostName); if (instanceSet != null) { instanceSet.remove(instance); if (instanceSet.isEmpty()) { allInstancesByHost.remove(instanceHostName); } } }
public int getTotalNumberOfSlots() { int count = 0; synchronized (globalLock) { for (Instance instance : allInstances) { if (instance.isAlive()) { count += instance.getTotalNumberOfSlots(); } } } return count; }
private void processNewlyAvailableInstances() { synchronized (globalLock) { Instance instance; while ((instance = newlyAvailableInstances.poll()) != null) { if (instance.hasResourcesAvailable()) { instancesWithAvailableResources.put(instance.getTaskManagerID(), instance); } } } }
/** * Unregisters the TaskManager with the given instance id. Unregistering means to mark * the given instance as dead and notify {@link InstanceListener} about the dead instance. * * @param instanceId TaskManager which is about to be marked dead. */ public void unregisterTaskManager(InstanceID instanceId, boolean terminated){ Instance instance = registeredHostsById.get(instanceId); if (instance != null){ registeredHostsById.remove(instance.getId()); registeredHostsByResource.remove(instance.getTaskManagerID()); if (terminated) { deadHosts.add(instance.getTaskManagerID()); } instance.markDead(); totalNumberOfAliveTaskSlots -= instance.getTotalNumberOfSlots(); notifyDeadInstance(instance); LOG.info( "Unregistered task manager " + instance.getTaskManagerLocation().addressString() + ". Number of registered task managers " + getNumberOfRegisteredTaskManagers() + ". Number of available slots " + getTotalNumberOfSlots() + "."); } else { LOG.warn("Tried to unregister instance {} but it is not registered.", instanceId); } }
throw new IllegalArgumentException(); if (instance.getNumberOfAvailableSlots() <= 0) { throw new IllegalArgumentException("The given instance has no resources."); if (!instance.isAlive()) { throw new IllegalArgumentException("The instance is not alive."); instance.setSlotAvailabilityListener(this); String instanceHostName = instance.getTaskManagerLocation().getHostname(); Set<Instance> instanceSet = allInstancesByHost.get(instanceHostName); if (instanceSet == null) { this.instancesWithAvailableResources.put(instance.getTaskManagerID(), instance); for (int i = 0; i < instance.getNumberOfAvailableSlots(); i++) { newSlotAvailable(instance);
SimpleSlot slot = instanceToUse.allocateSimpleSlot(); if (instanceToUse.hasResourcesAvailable()) { this.instancesWithAvailableResources.put(instanceToUse.getTaskManagerID(), instanceToUse);
gen.writeStringField("id", instance.getId().toString()); gen.writeStringField("path", instance.getTaskManagerGateway().getAddress()); gen.writeNumberField("dataPort", instance.getTaskManagerLocation().dataPort()); gen.writeNumberField("timeSinceLastHeartbeat", instance.getLastHeartBeat()); gen.writeNumberField("slotsNumber", instance.getTotalNumberOfSlots()); gen.writeNumberField("freeSlots", instance.getNumberOfAvailableSlots()); gen.writeNumberField("cpuCores", instance.getResources().getNumberOfCPUCores()); gen.writeNumberField("physicalMemory", instance.getResources().getSizeOfPhysicalMemory()); gen.writeNumberField("freeMemory", instance.getResources().getSizeOfJvmHeap()); gen.writeNumberField("managedMemory", instance.getResources().getSizeOfManagedMemory()); MetricStore.TaskManagerMetricStore metrics = fetcher.getMetricStore().getTaskManagerMetricStore(instance.getId().toString()); if (metrics != null) { gen.writeObjectFieldStart("metrics");
gen.writeStringField("id", instance.getTaskManagerID().getResourceIdString()); gen.writeStringField("path", instance.getTaskManagerGateway().getAddress()); gen.writeNumberField("dataPort", instance.getTaskManagerLocation().dataPort()); gen.writeNumberField("timeSinceLastHeartbeat", instance.getLastHeartBeat()); gen.writeNumberField("slotsNumber", instance.getTotalNumberOfSlots()); gen.writeNumberField("freeSlots", instance.getNumberOfAvailableSlots()); gen.writeNumberField("cpuCores", instance.getResources().getNumberOfCPUCores()); gen.writeNumberField("physicalMemory", instance.getResources().getSizeOfPhysicalMemory()); gen.writeNumberField("freeMemory", instance.getResources().getSizeOfJvmHeap()); gen.writeNumberField("managedMemory", instance.getResources().getSizeOfManagedMemory()); gen.writeEndObject(); fetcher.update(); MetricStore.TaskManagerMetricStore metrics = fetcher.getMetricStore().getTaskManagerMetricStore(instance.getTaskManagerID().getResourceIdString()); if (metrics != null) { gen.writeObjectFieldStart("metrics");
throw new IllegalStateException("Registration attempt from TaskManager at " + taskManagerLocation.addressString() + ". This connection is already registered under ID " + prior.getId()); Instance host = new Instance( taskManagerGateway, taskManagerLocation, host.reportHeartBeat();
SharedSlot sharedSlot = instanceToUse.allocateSharedSlot(groupAssignment); if (instanceToUse.hasResourcesAvailable()) { this.instancesWithAvailableResources.put(instanceToUse.getTaskManagerID(), instanceToUse);
@Override public void onSuccess(Object result) throws Throwable { Iterable<Instance> taskManagers = ((JobManagerMessages.RegisteredTaskManagers) result).asJavaIterable(); List<String> activeTaskManagers = new ArrayList<>(); for (Instance taskManager : taskManagers) { activeTaskManagers.add(taskManager.getId().toString()); String taskManagerPath = taskManager.getTaskManagerGateway().getAddress(); String queryServicePath = taskManagerPath.substring(0, taskManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME + "_" + taskManager.getTaskManagerID().getResourceIdString(); ActorRef taskManagerQueryService = actorSystem.actorFor(queryServicePath); queryMetrics(taskManagerQueryService); } synchronized (metrics) { // remove all metrics belonging to unregistered task managers metrics.taskManagers.keySet().retainAll(activeTaskManagers); } } }, ctx);
/** * Shuts the scheduler down. After shut down no more tasks can be added to the scheduler. */ public void shutdown() { synchronized (globalLock) { for (Instance i : allInstances) { i.removeSlotListener(); i.cancelAndReleaseAllSlots(); } allInstances.clear(); allInstancesByHost.clear(); instancesWithAvailableResources.clear(); taskQueue.clear(); } }
@Override public CompletableFuture<Collection<Tuple2<ResourceID, String>>> requestTaskManagerMetricQueryServicePaths(Time timeout) { return requestTaskManagerInstances(timeout) .thenApply( (Collection<Instance> instances) -> instances .stream() .map( (Instance instance) -> { final String taskManagerAddress = instance.getTaskManagerGateway().getAddress(); final String taskManagerMetricQuerServicePath = taskManagerAddress.substring(0, taskManagerAddress.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME + '_' + instance.getTaskManagerID().getResourceIdString(); return Tuple2.of(instance.getTaskManagerID(), taskManagerMetricQuerServicePath); }) .collect(Collectors.toList())); }
@VisibleForTesting @Nullable public Instance getInstance(ResourceID resourceId) { for (Instance instance : allInstances) { if (Objects.equals(resourceId, instance.getTaskManagerID())) { return instance; } } return null; } }
@Override public Future<BlobKey> apply(JobManagerMessages.TaskManagerInstance value) { Instance taskManager = value.instance().get(); if (serveLogFile) { return taskManager.getTaskManagerGateway().requestTaskManagerLog(timeTimeout); } else { return taskManager.getTaskManagerGateway().requestTaskManagerStdout(timeTimeout); } } });
/** * Unregisters the TaskManager with the given instance id. Unregistering means to mark * the given instance as dead and notify {@link InstanceListener} about the dead instance. * * @param instanceId TaskManager which is about to be marked dead. */ public void unregisterTaskManager(InstanceID instanceId, boolean terminated){ Instance instance = registeredHostsById.get(instanceId); if (instance != null){ registeredHostsById.remove(instance.getId()); registeredHostsByResource.remove(instance.getTaskManagerID()); if (terminated) { deadHosts.add(instance.getTaskManagerID()); } instance.markDead(); totalNumberOfAliveTaskSlots -= instance.getTotalNumberOfSlots(); notifyDeadInstance(instance); LOG.info( "Unregistered task manager " + instance.getTaskManagerLocation().addressString() + ". Number of registered task managers " + getNumberOfRegisteredTaskManagers() + ". Number of available slots " + getTotalNumberOfSlots() + "."); } else { LOG.warn("Tried to unregister instance {} but it is not registered.", instanceId); } }
throw new IllegalArgumentException(); if (instance.getNumberOfAvailableSlots() <= 0) { throw new IllegalArgumentException("The given instance has no resources."); if (!instance.isAlive()) { throw new IllegalArgumentException("The instance is not alive."); instance.setSlotAvailabilityListener(this); String instanceHostName = instance.getTaskManagerLocation().getHostname(); Set<Instance> instanceSet = allInstancesByHost.get(instanceHostName); if (instanceSet == null) { this.instancesWithAvailableResources.put(instance.getTaskManagerID(), instance); for (int i = 0; i < instance.getNumberOfAvailableSlots(); i++) { newSlotAvailable(instance);
SimpleSlot slot = instanceToUse.allocateSimpleSlot(); if (instanceToUse.hasResourcesAvailable()) { this.instancesWithAvailableResources.put(instanceToUse.getTaskManagerID(), instanceToUse);