public static String getValue(final String name, final Environment env) { return env.getVariablesList().stream() .filter(var -> var.getName().equals(name)) .findFirst() .map(var -> var.getValue()) .orElse(""); }
/** * Returns the value of the provided environment variable, or an empty {@link Optional} if no matching environment * variable was found. */ public static Optional<String> getEnvVar(Environment environment, String key) { return environment.getVariablesList().stream() .filter(v -> v.getName().equals(key)) .map(v -> v.getValue()) .findFirst(); }
public static Map<String, String> toMap(final Environment environment) { return environment.getVariablesList().stream() .collect( Collectors.toMap(var -> var.getName(), var -> var.getValue())); }
Protos.Environment.Variable variable = portEnvVars.get(0); Assert.assertEquals(variable.getName(), expectedPortEnvVar); Assert.assertEquals(variable.getValue(), requestedPort.toString());
@Test public void testUpdateDynamicToStaticPort() throws Exception { // Launch for the first time: get port 555 from dynamic port Resource reserveResource = recordLaunchWithCompleteOfferedResources( PodInstanceRequirementTestUtils.getPortRequirement(0), ResourceTestUtils.getUnreservedPorts(555, 555)).get(3); String resourceId = getResourceId(reserveResource); Collection<Resource> expectedResources = getExpectedExecutorResources( stateStore.fetchTasks().iterator().next().getExecutor()); expectedResources.addAll(Arrays.asList( ResourceTestUtils.getReservedPorts(555, 555, resourceId), ResourceTestUtils.getUnreservedPorts(666, 666))); // Now lets move to port 666: List<OfferRecommendation> recommendations = evaluator.evaluate( PodInstanceRequirementTestUtils.getPortRequirement(666), Arrays.asList(OfferTestUtils.getOffer(expectedResources))); // UNRESERVE, RESERVE, LAUNCH Assert.assertEquals(Arrays.asList( Protos.Offer.Operation.Type.UNRESERVE, Protos.Offer.Operation.Type.RESERVE, Offer.Operation.Type.LAUNCH_GROUP, null), recommendations.stream() .map(rec -> rec.getOperation().isPresent() ? rec.getOperation().get().getType() : null) .collect(Collectors.toList())); Operation launchOperation = recommendations.get(2).getOperation().get(); TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0); Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment()); Assert.assertEquals(String.valueOf(666), envvars.get(TestConstants.PORT_ENV_NAME + "_666").getValue()); }
@Test public void testUpdateStaticToStaticPort() throws Exception { // Launch for the first time: get port 555 Resource reserveResource = recordLaunchWithCompleteOfferedResources( PodInstanceRequirementTestUtils.getPortRequirement(555), ResourceTestUtils.getUnreservedPorts(555, 555)).get(3); String resourceId = getResourceId(reserveResource); Collection<Resource> expectedResources = getExpectedExecutorResources( stateStore.fetchTasks().iterator().next().getExecutor()); expectedResources.addAll(Arrays.asList( ResourceTestUtils.getReservedPorts(555, 555, resourceId), ResourceTestUtils.getUnreservedPorts(666, 666))); // Now lets move to port 666: List<OfferRecommendation> recommendations = evaluator.evaluate( PodInstanceRequirementTestUtils.getPortRequirement(666), Arrays.asList(OfferTestUtils.getOffer(expectedResources))); // UNRESERVE, RESERVE, LAUNCH Assert.assertEquals(Arrays.asList( Protos.Offer.Operation.Type.UNRESERVE, Protos.Offer.Operation.Type.RESERVE, Offer.Operation.Type.LAUNCH_GROUP, null), recommendations.stream() .map(rec -> rec.getOperation().isPresent() ? rec.getOperation().get().getType() : null) .collect(Collectors.toList())); Operation launchOperation = recommendations.get(2).getOperation().get(); TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0); Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment()); Assert.assertEquals(String.valueOf(666), envvars.get(TestConstants.PORT_ENV_NAME + "_666").getValue()); }
@Test public void testHealthCheckPortEnvvarIsCorrectOnOverlay() throws Exception { DefaultPodInstance podInstance = getPodInstance("valid-port-healthcheck-overlay.yml"); PodInstanceRequirement podInstanceRequirement = PodInstanceRequirement.newBuilder(podInstance, TaskUtils.getTaskNames(podInstance)) .build(); PodInfoBuilder podInfoBuilder = getPodInfoBuilder(podInstanceRequirement); Protos.Resource offeredPorts = ResourceTestUtils.getUnreservedPorts(10000, 10000); Protos.Offer offer = OfferTestUtils.getOffer(offeredPorts); PortEvaluationStage portEvaluationStage = new PortEvaluationStage( getPortSpec(podInstance), Collections.singleton(TestConstants.TASK_NAME), Optional.empty(), Optional.empty()); EvaluationOutcome outcome = portEvaluationStage.evaluate( new MesosResourcePool(offer, Optional.of(Constants.ANY_ROLE)), podInfoBuilder); Assert.assertTrue(outcome.isPassing()); Assert.assertEquals(0, outcome.getOfferRecommendations().size()); Protos.TaskInfo.Builder taskBuilder = podInfoBuilder.getTaskBuilders().stream().findFirst().get(); Assert.assertTrue(taskBuilder.getCommand().getEnvironment().getVariablesList().stream() .filter(variable -> variable.getName().equals("PORT_TEST_PORT") && variable.getValue().equals("10000")) .count() == 1); Assert.assertTrue(taskBuilder.getHealthCheck().getCommand().getEnvironment().getVariablesList().stream() .filter(variable -> variable.getName().equals("PORT_TEST_PORT") && variable.getValue().equals("10000")) .count() == 1); }
@SuppressWarnings("PMD.AvoidUsingHardCodedIP") @Test public void testReserveTaskNamedVIPPort() throws Exception { List<OfferRecommendation> recommendations = evaluator.evaluate( PodInstanceRequirementTestUtils.getVIPRequirement(80, 10000), OfferTestUtils.getCompleteOffers(ResourceTestUtils.getUnreservedPorts(10000, 10000))); Assert.assertEquals(6, recommendations.size()); Operation launchOperation = recommendations.get(4).getOperation().get(); Assert.assertFalse(recommendations.get(5).getOperation().isPresent()); TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0); Resource fulfilledPortResource = taskInfo.getResources(0); Assert.assertEquals(10000, fulfilledPortResource.getRanges().getRange(0).getBegin()); Assert.assertFalse(getResourceId(fulfilledPortResource).isEmpty()); DiscoveryInfo discoveryInfo = taskInfo.getDiscovery(); Assert.assertEquals(discoveryInfo.getName(), taskInfo.getName()); Assert.assertEquals(discoveryInfo.getVisibility(), DiscoveryInfo.Visibility.CLUSTER); Port discoveryPort = discoveryInfo.getPorts().getPorts(0); Assert.assertEquals(discoveryPort.getProtocol(), "tcp"); Assert.assertEquals(discoveryPort.getVisibility(), DiscoveryInfo.Visibility.EXTERNAL); Assert.assertEquals(discoveryPort.getNumber(), 10000); Label vipLabel = discoveryPort.getLabels().getLabels(0); Assert.assertTrue(vipLabel.getKey().startsWith("VIP_")); Assert.assertEquals(vipLabel.getValue(), TestConstants.VIP_NAME + "-10000:80"); Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment()); Assert.assertEquals(String.valueOf(10000), envvars.get(TestConstants.PORT_ENV_NAME + "_VIP_10000").getValue()); }
@Test public void testReadinessCheckPortEnvvarIsCorrectOnOverlay() throws Exception { DefaultPodInstance podInstance = getPodInstance("valid-port-readinesscheck-overlay.yml"); PodInstanceRequirement podInstanceRequirement = PodInstanceRequirement.newBuilder(podInstance, TaskUtils.getTaskNames(podInstance)) .build(); PodInfoBuilder podInfoBuilder = getPodInfoBuilder(podInstanceRequirement); Protos.Resource offeredPorts = ResourceTestUtils.getUnreservedPorts(10000, 10000); Protos.Offer offer = OfferTestUtils.getCompleteOffer(offeredPorts); PortEvaluationStage portEvaluationStage = new PortEvaluationStage( getPortSpec(podInstance), Collections.singleton(TestConstants.TASK_NAME), Optional.empty(), Optional.empty()); EvaluationOutcome outcome = portEvaluationStage.evaluate( new MesosResourcePool(offer, Optional.of(Constants.ANY_ROLE)), podInfoBuilder); Assert.assertTrue(outcome.isPassing()); Assert.assertEquals(0, outcome.getOfferRecommendations().size()); Protos.TaskInfo.Builder taskBuilder = podInfoBuilder.getTaskBuilders().stream().findFirst().get(); Assert.assertTrue(taskBuilder.getCommand().getEnvironment().getVariablesList().stream() .filter(variable -> variable.getName().equals("PORT_TEST_PORT") && variable.getValue().equals("10000")) .count() == 1); Protos.CheckInfo readinessCheck = taskBuilder.hasCheck() ? taskBuilder.getCheck() : null; Assert.assertTrue(readinessCheck != null); Assert.assertTrue(readinessCheck.getCommand().getCommand().getEnvironment().getVariablesList().stream() .filter(variable -> variable.getName().equals("PORT_TEST_PORT") && variable.getValue().equals("10000")) .count() == 1); }
@SuppressWarnings("PMD.AvoidUsingHardCodedIP") @Test public void testReserveTaskDynamicVIPPort() throws Exception { List<OfferRecommendation> recommendations = evaluator.evaluate( PodInstanceRequirementTestUtils.getVIPRequirement(80, 0), OfferTestUtils.getCompleteOffers(ResourceTestUtils.getUnreservedPorts(10000, 10000))); Assert.assertEquals(6, recommendations.size()); Operation launchOperation = recommendations.get(4).getOperation().get(); Assert.assertFalse(recommendations.get(5).getOperation().isPresent()); TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0); Resource fulfilledPortResource = taskInfo.getResources(0); Assert.assertEquals(10000, fulfilledPortResource.getRanges().getRange(0).getBegin()); Assert.assertFalse(getResourceId(fulfilledPortResource).isEmpty()); DiscoveryInfo discoveryInfo = taskInfo.getDiscovery(); Assert.assertEquals(discoveryInfo.getName(), taskInfo.getName()); Assert.assertEquals(discoveryInfo.getVisibility(), DiscoveryInfo.Visibility.CLUSTER); Port discoveryPort = discoveryInfo.getPorts().getPorts(0); Assert.assertEquals(discoveryPort.getProtocol(), "tcp"); Assert.assertEquals(discoveryPort.getVisibility(), DiscoveryInfo.Visibility.EXTERNAL); Assert.assertEquals(discoveryPort.getNumber(), 10000); Label vipLabel = discoveryPort.getLabels().getLabels(0); Assert.assertTrue(vipLabel.getKey().startsWith("VIP_")); Assert.assertEquals(vipLabel.getValue(), TestConstants.VIP_NAME + "-0:80"); Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment()); Assert.assertEquals(String.valueOf(10000), envvars.get(TestConstants.PORT_ENV_NAME + "_VIP_0").getValue()); }
@Test public void testReserveStaticPort() throws Exception { PodInstanceRequirement podInstanceRequirement = PodInstanceRequirementTestUtils.getPortRequirement(555); Protos.Resource offeredPorts = ResourceTestUtils.getUnreservedPorts(555, 555); List<OfferRecommendation> recommendations = evaluator.evaluate(podInstanceRequirement, OfferTestUtils.getCompleteOffers(offeredPorts)); Assert.assertEquals(Arrays.asList( Protos.Offer.Operation.Type.RESERVE, Protos.Offer.Operation.Type.RESERVE, Protos.Offer.Operation.Type.RESERVE, Protos.Offer.Operation.Type.RESERVE, Offer.Operation.Type.LAUNCH_GROUP, null), recommendations.stream() .map(rec -> rec.getOperation().isPresent() ? rec.getOperation().get().getType() : null) .collect(Collectors.toList())); Protos.Offer.Operation launchOperation = recommendations.get(4).getOperation().get(); Protos.TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0); Protos.Resource fulfilledPortResource = taskInfo.getResources(0); Assert.assertFalse(getResourceId(fulfilledPortResource).isEmpty()); Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment()); Assert.assertEquals(String.valueOf(555), envvars.get(TestConstants.PORT_ENV_NAME + "_555").getValue()); }
@Override public void expect(ClusterState state, SchedulerDriver mockDriver) throws AssertionError { StateStore stateStore = new StateStore(persisterWithTasks); Optional<Protos.TaskInfo> task = stateStore.fetchTask(taskName); Assert.assertTrue( String.format("Missing task: %s, known tasks are: %s", taskName, stateStore.fetchTaskNames()), task.isPresent()); Collection<Protos.Environment.Variable> env = task.get().getCommand().getEnvironment().getVariablesList(); Optional<String> actualValue = env.stream() .filter(e -> e.getName().equals(key)) .map(e -> e.getValue()) .findFirst(); Assert.assertTrue(String.format("Missing env entry %s, known entries are: %s", key, env.stream().map(e -> TextFormat.shortDebugString(e)).collect(Collectors.toList())), actualValue.isPresent()); Assert.assertEquals(value, actualValue.get()); } };
@Test public void ensureUnder1GBIsLessThan256MB() throws Exception { int ram = 512; Mockito.when(configuration.getMem()).thenReturn((double) ram); ExecutorEnvironmentalVariables env = new ExecutorEnvironmentalVariables(configuration); for (Protos.Environment.Variable var : env.getList()) { if (var.getName().equals(ExecutorEnvironmentalVariables.ES_HEAP)) { String val = var.getValue(); Pattern pattern = Pattern.compile("(\\d*)m"); Matcher matcher = pattern.matcher(val); assertTrue(matcher.matches()); assertEquals(Integer.toString(ram - ram / 4), matcher.group(1)); assertTrue(ram - Integer.parseInt(matcher.group(1)) < 256); assertTrue(ram - Integer.parseInt(matcher.group(1)) < ram); assertTrue(ram - Integer.parseInt(matcher.group(1)) > 0); } } } }
@Test public void addEnvVarToEnvironmentWithSecret() { Protos.Environment.Builder envBuilder = Protos.Environment.newBuilder(); envBuilder.addVariablesBuilder() .setName(SECRET_KEY) .setType(Protos.Environment.Variable.Type.SECRET) .setSecret(getReferenceSecret(SECRET_PATH)); Protos.Environment newEnv = EnvUtils.withEnvVar(envBuilder.build(), TEST_ENV_KEY, TEST_ENV_VALUE); Assert.assertEquals(newEnv.getVariablesCount(), 2); for(Protos.Environment.Variable envVar : newEnv.getVariablesList()) { if (envVar.getName().equals(SECRET_KEY)) { Assert.assertEquals(envVar.getSecret(), getReferenceSecret(SECRET_PATH)); } if (envVar.getName().equals(TEST_ENV_KEY)) { Assert.assertEquals(envVar.getValue(), TEST_ENV_VALUE); } } }
@Test public void testReserveTaskDynamicPort() throws Exception { Protos.Resource offeredPorts = ResourceTestUtils.getUnreservedPorts(10000, 10000); PodInstanceRequirement podInstanceRequirement = PodInstanceRequirementTestUtils.getPortRequirement(0); List<OfferRecommendation> recommendations = evaluator.evaluate(podInstanceRequirement, OfferTestUtils.getCompleteOffers(offeredPorts)); Assert.assertEquals(6, recommendations.size()); Protos.Offer.Operation launchOperation = recommendations.get(4).getOperation().get(); Assert.assertFalse(recommendations.get(5).getOperation().isPresent()); Protos.TaskInfo taskInfo = launchOperation.getLaunchGroup().getTaskGroup().getTasks(0); Protos.Resource fulfilledPortResource = taskInfo.getResources(0); Assert.assertFalse(getResourceId(fulfilledPortResource).isEmpty()); Map<String, Protos.Environment.Variable> envvars = EnvUtils.toMap(taskInfo.getCommand().getEnvironment()); Assert.assertEquals(envvars.toString(), String.valueOf(10000), envvars.get(TestConstants.PORT_ENV_NAME + "_0").getValue()); }
@Test public void ensureOver1GBHeapIs256MB() throws Exception { int ram = 2048; Mockito.when(configuration.getMem()).thenReturn((double) ram); ExecutorEnvironmentalVariables env = new ExecutorEnvironmentalVariables(configuration); for (Protos.Environment.Variable var : env.getList()) { if (var.getName().equals(ExecutorEnvironmentalVariables.ES_HEAP)) { String val = var.getValue(); Pattern pattern = Pattern.compile("(\\d*)m"); Matcher matcher = pattern.matcher(val); assertTrue(matcher.matches()); assertEquals(Integer.toString(ram - 256), matcher.group(1)); } } }
/** * Gets the zone of a task. * * @param taskInfo The {@link TaskInfo} to get zone information from. * @return A string indicating the zone the task is in. */ public static String getTaskZone(Protos.TaskInfo taskInfo) { return taskInfo.getCommand().getEnvironment().getVariablesList().stream() .filter(variable -> variable.getName().equals(EnvConstants.ZONE_TASKENV)).findFirst().get().getValue(); }
@Test public void regionAndZoneInjected() { offer = offer.toBuilder() .setDomain(TestConstants.LOCAL_DOMAIN_INFO) .build(); stage.evaluate(new MesosResourcePool(offer, Optional.of(Constants.ANY_ROLE)), podInfoBuilder); Protos.TaskInfo.Builder taskBuilder = podInfoBuilder.getTaskBuilder(TestConstants.TASK_NAME); Map<String, Protos.Environment.Variable> env = EnvUtils.toMap(taskBuilder.getCommand().getEnvironment()); Assert.assertEquals(TestConstants.LOCAL_REGION, env.get(EnvConstants.REGION_TASKENV).getValue()); Assert.assertEquals(TestConstants.ZONE, env.get(EnvConstants.ZONE_TASKENV).getValue()); }
@Test public void overwriteExistingEnvVarKey() { Protos.Environment.Builder envBuilder = Protos.Environment.newBuilder(); envBuilder.addVariablesBuilder() .setName(TEST_ENV_KEY) .setValue(TEST_ENV_VALUE) .build(); Protos.Environment newEnv = EnvUtils.withEnvVar(envBuilder.build(), TEST_ENV_KEY, TEST_REPLACE_ENV_VALUE); Assert.assertEquals(newEnv.getVariablesCount(), 1); Protos.Environment.Variable envVar = newEnv.getVariables(0); Assert.assertEquals(envVar.getValue(), TEST_REPLACE_ENV_VALUE); }