LaunchedTask task = state.getLastLaunchedTask(taskName); expectedResourceIds.addAll(ResourceUtils.getResourceIds(task.getTask().getResourcesList())); expectedResourceIds.addAll(ResourceUtils.getResourceIds(task.getExecutor().getResourcesList()));
/** * Returns a list of all the resources associated with a task, including {@link Executor} resources. * * @param taskInfo The {@link Protos.TaskInfo} containing the {@link Protos.Resource}. * @return a list of {@link Protos.Resource}s. */ public static List<Protos.Resource> getAllResources(Protos.TaskInfo taskInfo) { // Get all resources from both the task level and the executor level List<Protos.Resource> resources = new ArrayList<>(taskInfo.getResourcesList()); if (taskInfo.hasExecutor()) { resources.addAll(taskInfo.getExecutor().getResourcesList()); } return resources; }
private static Map<String, Status> getExpectedStepStatuses(ClusterState state, StepCount stepCount) { Map<String, Status> expectedSteps = new TreeMap<>(); expectedSteps.put(String.format("kill-%s-server", stepCount.phaseName), stepCount.statusOfStepIndex(expectedSteps.size())); List<AcceptEntry> acceptCalls = state.getAcceptCalls(stepCount.phaseName); // Get information based on the LAST operation. This wouldn't work if the last operation didn't reserve // anything, as would occur when a task is restarted, but this is close enough for our purposes. AcceptEntry acceptCall = acceptCalls.get(acceptCalls.size() - 1); Collection<String> resourceIds = new ArrayList<>(); resourceIds.addAll(ResourceUtils.getResourceIds(ResourceUtils.getAllResources(acceptCall.getTasks()))); resourceIds.addAll(acceptCall.getExecutors().stream() .map(e -> ResourceUtils.getResourceIds(e.getResourcesList())) .flatMap(Collection::stream) .collect(Collectors.toList())); for (String resourceId : resourceIds) { expectedSteps.put(String.format("unreserve-%s", resourceId), stepCount.statusOfStepIndex(expectedSteps.size())); } expectedSteps.put(String.format("erase-%s-server", stepCount.phaseName), stepCount.statusOfStepIndex(expectedSteps.size())); return expectedSteps; } }
resourceSpecForRoleAndPrincipal.getPrincipal(), resourceSpecForRoleAndPrincipal.getPreReservedRole()), executorInfo.getResourcesList(), resourceNamespace); executorResourceMapper.getOrphanedResources()
filterResources(taskInfo.getResourcesList(), resourceIdsToRemove); Collection<Protos.Resource> updatedExecutorResources = filterResources(taskInfo.getExecutor().getResourcesList(), resourceIdsToRemove); if (updatedTaskResources == null && updatedExecutorResources == null) {
private Collection<Resource> getExpectedExecutorResources(Protos.ExecutorInfo executorInfo) { String executorCpuId = executorInfo.getResourcesList().stream() .filter(r -> r.getName().equals("cpus")) .map(ResourceUtils::getResourceId) .filter(o -> o.isPresent()) .map(o -> o.get()) .findFirst() .get(); String executorMemId = executorInfo.getResourcesList().stream() .filter(r -> r.getName().equals("mem")) .map(ResourceUtils::getResourceId) .filter(o -> o.isPresent()) .map(o -> o.get()) .findFirst() .get(); String executorDiskId = executorInfo.getResourcesList().stream() .filter(r -> r.getName().equals("disk")) .map(ResourceUtils::getResourceId) .filter(o -> o.isPresent()) .map(o -> o.get()) .findFirst() .get(); Resource expectedExecutorCpu = ResourceTestUtils.getReservedCpus(0.1, executorCpuId); Resource expectedExecutorMem = ResourceTestUtils.getReservedMem(32, executorMemId); Resource expectedExecutorDisk = ResourceTestUtils.getReservedDisk(256, executorDiskId); return new ArrayList<>(Arrays.asList(expectedExecutorCpu, expectedExecutorMem, expectedExecutorDisk)); } }
private Collection<Protos.Resource> getExpectedExecutorResources(Protos.ExecutorInfo executorInfo) { String executorCpuId = executorInfo.getResourcesList().stream() .filter(r -> r.getName().equals("cpus")) .map(ResourceUtils::getResourceId) .filter(o -> o.isPresent()) .map(o -> o.get()) .findFirst() .get(); String executorMemId = executorInfo.getResourcesList().stream() .filter(r -> r.getName().equals("mem")) .map(ResourceUtils::getResourceId) .filter(o -> o.isPresent()) .map(o -> o.get()) .findFirst() .get(); String executorDiskId = executorInfo.getResourcesList().stream() .filter(r -> r.getName().equals("disk")) .map(ResourceUtils::getResourceId) .filter(o -> o.isPresent()) .map(o -> o.get()) .findFirst() .get(); Protos.Resource expectedExecutorCpu = ResourceTestUtils.getReservedCpus(0.1, executorCpuId); Protos.Resource expectedExecutorMem = ResourceTestUtils.getReservedMem(32, executorMemId); Protos.Resource expectedExecutorDisk = ResourceTestUtils.getReservedDisk(256, executorDiskId); return new ArrayList<>(Arrays.asList(expectedExecutorCpu, expectedExecutorMem, expectedExecutorDisk)); }
/** * This method keeps the resources associated with tasks in the state store up to date, when a task which shares * their resource-set is launched. * * @param podInstance the parent pod associated with the task being launched * @param taskInfo the task being launched */ @VisibleForTesting void updateTaskResourcesWithinResourceSet(PodInstance podInstance, Protos.TaskInfo taskInfo) { Optional<TaskSpec> taskSpec = TaskUtils.getTaskSpec(podInstance, taskInfo.getName()); if (!taskSpec.isPresent()) { return; } // Update any other TaskInfos in this resource set to have the same resources: Collection<Protos.TaskInfo> taskInfosWithSameResourceSet = getOtherTasksInResourceSet(podInstance, taskSpec.get()); stateStore.storeTasks(updateTasksWithResources( taskInfosWithSameResourceSet, taskInfo.getResourcesList(), taskInfo.hasExecutor() ? Optional.of(taskInfo.getExecutor().getResourcesList()) : Optional.empty())); }
public static Protos.Offer generateUpdateOffer( String frameworkId, Protos.TaskInfo taskInfo, Protos.TaskInfo templateTaskInfo, double cpu, int memory, int disk) { final String offerUUID = UUID.randomUUID().toString(); return Protos.Offer .newBuilder() .setId(Protos.OfferID.newBuilder().setValue(offerUUID)) .setFrameworkId(Protos.FrameworkID.newBuilder().setValue(frameworkId)) .setSlaveId(Protos.SlaveID.newBuilder().setValue(taskInfo.getSlaveId().getValue())) .setHostname("127.0.0.1") .addAllResources(taskInfo.getResourcesList()) .addAllResources(taskInfo.getExecutor().getResourcesList()) .addAllResources(templateTaskInfo.getResourcesList()) .addResources(ResourceUtils.getUnreservedScalar("cpus", cpu)) .addResources(ResourceUtils.getUnreservedScalar("mem", memory)) .addResources(ResourceUtils.getUnreservedScalar("disk", disk)) .build(); }
public CassandraTaskExecutor update(final ExecutorConfig config) { return new CassandraTaskExecutor( Protos.ExecutorInfo.newBuilder(info) .setExecutorId(ExecutorUtils.toExecutorId(info.getName())) .setCommand(createCommandInfo(config.getCommand(), config.getArguments(), config.getURIs(), config.getCacheFetchedURIs(), ImmutableMap.<String, String>builder() .put("JAVA_HOME", config.getJavaHome()) .put("JAVA_OPTS", "-Xmx" + config.getHeapMb() + "M") .put("EXECUTOR_API_PORT", Integer.toString(config.getApiPort())) .build())) .clearResources() .addAllResources( updateResources( config.getCpus(), config.getMemoryMb(), info.getResourcesList())).build()); }
public static Protos.Offer generateReplacementOffer( String frameworkId, Protos.TaskInfo taskInfo, Protos.TaskInfo templateTaskInfo) { final String offerUUID = UUID.randomUUID().toString(); return Protos.Offer .newBuilder() .setId(Protos.OfferID.newBuilder().setValue(offerUUID)) .setFrameworkId(Protos.FrameworkID.newBuilder().setValue(frameworkId)) .setSlaveId(Protos.SlaveID.newBuilder().setValue(taskInfo.getSlaveId().getValue())) .setHostname("127.0.0.1") .addAllResources(taskInfo.getResourcesList()) .addAllResources(taskInfo.getExecutor().getResourcesList()) .addAllResources(templateTaskInfo.getResourcesList()) .build(); }
private static List<Protos.Resource> getExpectedResources(Collection<OfferRecommendation> operations) { for (OfferRecommendation operation : operations) { if (operation.getOperation().get().getType().equals(Offer.Operation.Type.LAUNCH_GROUP)) { return Stream.concat( operation.getOperation().get().getLaunchGroup().getTaskGroup().getTasksList().stream() .flatMap(taskInfo -> taskInfo.getResourcesList().stream()), operation.getOperation().get().getLaunchGroup().getExecutor().getResourcesList().stream()) .collect(Collectors.toList()); } } return Collections.emptyList(); }
/** * Gets the executors memory allocation. * * @return The memory allocated to the executor in Mb. */ public int getMemoryMb() { return getResourceMemoryMb(info.getResourcesList()); }
/** * Gets the cpu shares. * * @return The cpu shares allocated to the executor. */ public double getCpus() { return getResourceCpus(info.getResourcesList()); }
private void validateDaemonExecutorInfo(Protos.ExecutorInfo executorInfo) { for (Protos.Resource resource : executorInfo.getResourcesList()) { Assert.assertTrue(ResourceUtils.getResourceId(resource).isEmpty()); } }