case SCALAR: resources.add(resource); aggregatedScalarResourceMap.merge(resource.getName(), resource.getScalar().getValue(), Double::sum); break; case RANGES:
double amountToTake = Math.min(available.getScalar().getValue(), amount); Protos.Resource taken = available.toBuilder().setScalar(Protos.Value.Scalar.newBuilder().setValue(amountToTake)).build(); amount -= amountToTake; double remaining = available.getScalar().getValue() - taken.getScalar().getValue(); if (remaining > EPSILON) { i.set(available.toBuilder().setScalar(Protos.Value.Scalar.newBuilder().setValue(remaining)).build());
if (resource.getName().equals(CPU)) { if (resource.getType().equals(Protos.Value.Type.SCALAR)) cpus = resource.getScalar().getValue(); else log.log(Level.FINE, "Cpus resource was not a scalar: {0}" + resource.getType()); mem = resource.getScalar().getValue(); else log.log(Level.FINE, "Mem resource was not a scalar: {0}", resource.getType()); disk = resource.getScalar().getValue(); else log.log(Level.FINE, "Disk resource was not a scalar: {0}", resource.getType());
private void testResourceDecrease(Optional<String> namespace) { String resourceId = UUID.randomUUID().toString(); Protos.Value current = getValue(5); Protos.Value desired = getValue(4); Protos.Value toSubtract = ValueUtils.subtract(current, desired); ResourceSpec resourceSpec = getResourceSpec(desired); when(mockPool.consumeReserved(RESOURCE_NAME, desired, resourceId)) .thenReturn(Optional.of(getMesosResource(getResourceSpec(current), resourceId, namespace))); when(mockPool.consumeReservableMerged(RESOURCE_NAME, desired, Constants.ANY_ROLE)) .thenReturn(Optional.of(getMesosResource(toSubtract))); ReserveEvaluationOutcome outcome = OfferEvaluationUtils.evaluateSimpleResource( LOGGER, mockStage, resourceSpec, Optional.of(resourceId), namespace, mockPool); Assert.assertTrue(outcome.getEvaluationOutcome().isPassing()); OfferRecommendation recommendation = outcome.getEvaluationOutcome().getOfferRecommendations().get(0); Assert.assertTrue(recommendation instanceof UnreserveOfferRecommendation); Assert.assertTrue(outcome.getResourceId().isPresent()); Protos.Resource resource = recommendation.getOperation().get().getUnreserve().getResources(0); Assert.assertEquals(toSubtract.getScalar(), resource.getScalar()); if (namespace.isPresent()) { Assert.assertEquals(namespace.get(), ResourceUtils.getNamespace(resource).get()); } else { Assert.assertFalse(ResourceUtils.getNamespace(resource).isPresent()); } }
private void testResourceIncreaseSufficient(Optional<String> namespace) { String resourceId = UUID.randomUUID().toString(); Protos.Value current = getValue(4); Protos.Value desired = getValue(5); Protos.Value toAdd = ValueUtils.subtract(desired, current); ResourceSpec resourceSpec = getResourceSpec(desired); when(mockPool.consumeReserved(RESOURCE_NAME, desired, resourceId)) .thenReturn(Optional.of(getMesosResource(getResourceSpec(current), resourceId, namespace))); when(mockPool.consumeReservableMerged(RESOURCE_NAME, toAdd, Constants.ANY_ROLE)) .thenReturn(Optional.of(getMesosResource(toAdd))); ReserveEvaluationOutcome outcome = OfferEvaluationUtils.evaluateSimpleResource( LOGGER, mockStage, resourceSpec, Optional.of(resourceId), namespace, mockPool); Assert.assertTrue(outcome.getEvaluationOutcome().isPassing()); OfferRecommendation recommendation = outcome.getEvaluationOutcome().getOfferRecommendations().get(0); Assert.assertTrue(recommendation instanceof ReserveOfferRecommendation); Assert.assertTrue(outcome.getResourceId().isPresent()); Protos.Resource resource = recommendation.getOperation().get().getReserve().getResources(0); Assert.assertEquals(toAdd.getScalar(), resource.getScalar()); if (namespace.isPresent()) { Assert.assertEquals(namespace.get(), ResourceUtils.getNamespace(resource).get()); } else { Assert.assertFalse(ResourceUtils.getNamespace(resource).isPresent()); } }
@Test public void testReserveLaunchScalar() throws Exception { PodInstanceRequirement podInstanceRequirement = PodInstanceRequirementTestUtils.getCpuRequirement(1.0); Protos.Resource offeredResource = ResourceTestUtils.getUnreservedCpus(2.0); List<OfferRecommendation> recommendations = evaluator.evaluate( podInstanceRequirement, Arrays.asList(OfferTestUtils.getCompleteOffer(offeredResource))); Assert.assertEquals(6, recommendations.size()); // Validate RESERVE Operation Protos.Offer.Operation reserveOperation = recommendations.get(3).getOperation().get(); Protos.Resource reserveResource = reserveOperation.getReserve().getResources(0); Protos.Resource.ReservationInfo reservation = ResourceUtils.getReservation(reserveResource).get(); Assert.assertEquals(Protos.Offer.Operation.Type.RESERVE, reserveOperation.getType()); Assert.assertEquals(1.0, reserveResource.getScalar().getValue(), 0.0); validateRole(reserveResource); Assert.assertEquals(TestConstants.ROLE, ResourceUtils.getRole(reserveResource)); Assert.assertEquals(TestConstants.PRINCIPAL, reservation.getPrincipal()); Assert.assertEquals(36, getResourceId(reserveResource).length()); Assert.assertFalse(reserveResource.hasDisk()); // Validate LAUNCH Operation Protos.Offer.Operation launchOperation = recommendations.get(4).getOperation().get(); Protos.Resource launchResource = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0).getResources(0); Assert.assertEquals(Protos.Offer.Operation.Type.LAUNCH_GROUP, launchOperation.getType()); Assert.assertEquals(getResourceId(reserveResource), getResourceId(launchResource)); Protos.ExecutorID executorId = launchOperation.getLaunchGroup().getExecutor().getExecutorId(); Assert.assertEquals(TestConstants.POD_TYPE, CommonIdUtils.toExecutorName(executorId)); }
private void parseOffer(Offer offer) { // Pull out the cpus, memory, disk, and 2 ports from the offer. for (Resource resource : offer.getResourcesList()) { if (resource.getName().equals("cpus") && resource.getType() == Value.Type.SCALAR) { cpus = resource.getScalar().getValue(); cpuRole = resource.getRole(); } else if (resource.getName().equals("mem") && resource.getType() == Value.Type.SCALAR) { mem = resource.getScalar().getValue(); memRole = resource.getRole(); } else if (resource.getName().equals("disk") && resource.getType() == Value.Type.SCALAR) { disk = resource.getScalar().getValue(); diskRole = resource.getRole(); } else if (resource.getName().equals("ports") && resource.getType() == Value.Type.RANGES) { portRole = resource.getRole(); ports = resource.getRanges().getRangeList(); } } }
@SuppressWarnings("deprecation") private static VolumeSpec getVolumeSpec(Protos.Resource resource) { return resource.getDisk().hasSource() ? DefaultVolumeSpec.createMountVolume( resource.getScalar().getValue(), resource.getDisk().getVolume().getContainerPath(), resource.getDisk().getSource().hasProfile() ? Collections.singletonList(resource.getDisk().getSource().getProfile()) : Collections.emptyList(), ResourceUtils.getRole(resource), resource.getRole(), resource.getDisk().getPersistence().getPrincipal()) : DefaultVolumeSpec.createRootVolume( resource.getScalar().getValue(), resource.getDisk().getVolume().getContainerPath(), ResourceUtils.getRole(resource), resource.getRole(), resource.getDisk().getPersistence().getPrincipal()); }
public static Value getValue(Resource resource) { Type type = resource.getType(); Value.Builder builder = Value.newBuilder(); builder.setType(type); switch (type) { case SCALAR: return builder.setScalar(resource.getScalar()).build(); case RANGES: return builder.setRanges(resource.getRanges()).build(); case SET: return builder.setSet(resource.getSet()).build(); default: throw new IllegalArgumentException(String.format("Unsupported value type %s in resource %s", type, TextFormat.shortDebugString(resource))); } }
@Test public void testConsumeUnreservedMergedResource() { Resource resource = ResourceTestUtils.getUnreservedCpus(1.0); Protos.Value resourceValue = ValueUtils.getValue(resource); Offer offer = OfferTestUtils.getOffer(resource); MesosResourcePool pool = new MesosResourcePool(offer, Optional.of(Constants.ANY_ROLE)); Assert.assertEquals(1, pool.getUnreservedMergedPool().size()); Assert.assertEquals(resource.getScalar().getValue(), pool.getUnreservedMergedPool().get("cpus").getScalar().getValue(), 0.0); MesosResource resourceToConsume = pool.consumeReservableMerged(resource.getName(), resourceValue, Constants.ANY_ROLE).get(); Assert.assertEquals(resource, resourceToConsume.getResource()); Assert.assertEquals(ValueUtils.getZero(Protos.Value.Type.SCALAR), pool.getUnreservedMergedPool().get("cpus")); }
static double scalarSum(Protos.Offer offer, String name) { return offer.getResourcesList().stream().filter(resource -> resource.getName().equals(name)).mapToDouble(resource -> resource.getScalar().getValue()).sum(); } }
private Double getScalar(List<Protos.Resource> resources, String name) { for (Protos.Resource resource : resources) { if (resource.getName().equals(name)) { return resource.getScalar().getValue(); } } return null; }
public static int getResourceMemoryMb(final List<Resource> resources) { return (int) Math.floor( resources.stream().filter(TaskUtils::isCpus) .map(resource -> resource.getScalar().getValue()) .reduce(0.0, Double::sum)); }
public static int getResourceDiskMb(final List<Resource> resources) { return (int) Math.floor( resources.stream().filter(TaskUtils::isCpus) .map(resource -> resource.getScalar().getValue()) .reduce(0.0, Double::sum)); }
@SuppressWarnings("deprecation") @Test public void testUnreservedResource() { ResourceBuilder resourceBuilder = ResourceBuilder.fromUnreservedValue("cpus", VALUE); Protos.Resource resource = resourceBuilder.build(); Assert.assertEquals("cpus", resource.getName()); Assert.assertEquals(Protos.Value.Type.SCALAR, resource.getType()); Assert.assertEquals(1.0, resource.getScalar().getValue(), 0.0); Assert.assertEquals(Constants.ANY_ROLE, resource.getRole()); Assert.assertFalse(resource.hasReservation()); Assert.assertEquals(0, resource.getReservationsCount()); }
public static double getResourceCpus(final List<Resource> resources) { return resources.stream().filter(TaskUtils::isCpus) .map(resource -> resource.getScalar().getValue()) .reduce(0.0, Double::sum); }
private int getCpu(final Offer offer) { for (final Resource resource : offer.getResourcesList()) { if (resource.getName().equals("cpus")) { return (int)resource.getScalar().getValue(); } } return 0; }
private int getMemory(final Offer offer) { for (final Resource resource : offer.getResourcesList()) { if (resource.getName().equals("mem")) { return (int)resource.getScalar().getValue(); } } return 0; }
/** * Gets a string representation of a resource. */ public static String toString(Protos.Resource resource) { checkNotNull(resource); if (resource.hasScalar()) { return String.format("%s(%s):%.1f", resource.getName(), resource.getRole(), resource.getScalar().getValue()); } if (resource.hasRanges()) { return String.format("%s(%s):%s", resource.getName(), resource.getRole(), toString(resource.getRanges())); } return resource.toString(); }