final String taskId = taskInfo.getTaskId().getValue();
/** {@inheritDoc} */ @Override public synchronized void resourceOffers(SchedulerDriver schedulerDriver, List<Protos.Offer> offers) { log.log(Level.FINE, "Offers resources: {0}", offers.size()); for (Protos.Offer offer : offers) { IgniteTask igniteTask = checkOffer(offer); // Decline offer which doesn't match by mem or cpu. if (igniteTask == null) { schedulerDriver.declineOffer(offer.getId()); continue; } // Generate a unique task ID. Protos.TaskID taskId = Protos.TaskID.newBuilder() .setValue(Integer.toString(taskIdGenerator.incrementAndGet())).build(); log.log(Level.INFO, "Launching task: {0}", igniteTask); // Create task to run. Protos.TaskInfo task = createTask(offer, igniteTask, taskId); try { schedulerDriver.launchTasks(Collections.singletonList(offer.getId()), Collections.singletonList(task), Protos.Filters.newBuilder().setRefuseSeconds(1).build()); } catch (RuntimeException e) { log.log(Level.SEVERE, "Failed launch task. Task id: {0}. Task info: {1}", new Object[]{taskId, task, e}); throw e; } tasks.put(taskId.getValue(), igniteTask); } }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.scheduler.Protos.Call.Acknowledge)) { return super.equals(obj); } org.apache.mesos.scheduler.Protos.Call.Acknowledge other = (org.apache.mesos.scheduler.Protos.Call.Acknowledge) obj; boolean result = true; result = result && (hasSlaveId() == other.hasSlaveId()); if (hasSlaveId()) { result = result && getSlaveId() .equals(other.getSlaveId()); } result = result && (hasTaskId() == other.hasTaskId()); if (hasTaskId()) { result = result && getTaskId() .equals(other.getTaskId()); } result = result && (hasUuid() == other.hasUuid()); if (hasUuid()) { result = result && getUuid() .equals(other.getUuid()); } result = result && unknownFields.equals(other.unknownFields); return result; }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.scheduler.Protos.Call.Kill)) { return super.equals(obj); } org.apache.mesos.scheduler.Protos.Call.Kill other = (org.apache.mesos.scheduler.Protos.Call.Kill) obj; boolean result = true; result = result && (hasTaskId() == other.hasTaskId()); if (hasTaskId()) { result = result && getTaskId() .equals(other.getTaskId()); } result = result && (hasSlaveId() == other.hasSlaveId()); if (hasSlaveId()) { result = result && getSlaveId() .equals(other.getSlaveId()); } result = result && (hasKillPolicy() == other.hasKillPolicy()); if (hasKillPolicy()) { result = result && getKillPolicy() .equals(other.getKillPolicy()); } result = result && unknownFields.equals(other.unknownFields); return result; }
@Test public void installAndRecover() throws Exception { install(); Collection<? extends Step> incompleteSteps = getIncompleteSteps(); assertTrue(incompleteSteps.isEmpty()); final CassandraDaemonTask task = cassandraState.getDaemons().get("node-0"); scheduler.statusUpdate(driver, TestUtils.generateStatus(task.getTaskInfo().getTaskId(), Protos.TaskState.TASK_KILLED)); Set<Protos.TaskStatus> taskStatuses = cassandraState.getTaskStatuses(); final Optional<Protos.TaskStatus> first = taskStatuses.stream().filter(status -> status.getTaskId().equals(task.getTaskInfo().getTaskId())).findFirst(); assertEquals(Protos.TaskState.TASK_KILLED, first.get().getState()); final CassandraTask templateTask = cassandraState.get("node-0-task-template").get(); final Protos.Offer offer = TestUtils.generateReplacementOffer(frameworkId.getValue(), task.getTaskInfo(), templateTask.getTaskInfo()); scheduler.resourceOffers(driver, Arrays.asList(offer)); Collection<QueuedSchedulerDriver.OfferOperations> offerOps = driver.drainAccepted(); assertEquals(String.format("expected accepted offer: %s", offer), 1, offerOps.size()); Collection<Protos.Offer.Operation> ops = offerOps.iterator().next().getOperations(); launchAll(ops, scheduler, driver); taskStatuses = cassandraState.getTaskStatuses(); final Optional<Protos.TaskStatus> node0Status = taskStatuses.stream().filter(status -> { try { return org.apache.mesos.offer.TaskUtils.toTaskName(status.getTaskId()).equals(task.getTaskInfo().getName()); } catch (Exception e) { throw new RuntimeException(e); } }).findFirst(); assertEquals(Protos.TaskState.TASK_RUNNING, node0Status.get().getState()); }
@Test public void testExecutorStateMechanism() throws IOException, InterruptedException, ExecutionException, ClassNotFoundException { Protos.TaskStatus taskStatus = Protos.TaskStatus.getDefaultInstance(); SerializableState state = Mockito.mock(SerializableState.class); when(state.get(anyString())).thenReturn(taskStatus); Protos.FrameworkID frameworkID = Protos.FrameworkID.newBuilder().setValue(FRAMEWORK_ID).build(); Protos.SlaveID slaveID = Protos.SlaveID.newBuilder().setValue(SLAVE_ID).build(); Protos.ExecutorID executorID = Protos.ExecutorID.newBuilder().setValue(EXECUTOR_ID).build(); Protos.TaskID taskID = Protos.TaskID.newBuilder().setValue(TASK_ID).build(); Protos.TaskInfo taskInfo = Protos.TaskInfo.newBuilder() .setTaskId(taskID) .setExecutor(Protos.ExecutorInfo.newBuilder() .setExecutorId(executorID) .setCommand(Protos.CommandInfo.getDefaultInstance()) ) .setSlaveId(slaveID) .setName("Test") .build(); ESTaskStatus executorState = new ESTaskStatus(state, frameworkID, taskInfo, mock(StatePath.class)); executorState.setStatus(taskStatus); verify(state, times(1)).set(anyString(), any(Protos.TaskStatus.class)); Protos.TaskStatus newStatus = executorState.getStatus(); assertNotNull(newStatus); assertEquals(taskStatus.getExecutorId().toString(), newStatus.getExecutorId().toString()); } }
private void onStop() { // Shutdown REEF Evaluator if (this.evaluatorProcess != null) { this.evaluatorProcess.destroy(); mesosExecutorDriver.sendStatusUpdate(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder() .setValue(mesosExecutorId) .build()) .setState(TaskState.TASK_FINISHED) .setMessage("Evaluator Process exited with status " + String.valueOf(evaluatorProcessExitValue)) .build()); } else { mesosExecutorDriver.sendStatusUpdate(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder() .setValue(mesosExecutorId) .build()) .setState(TaskState.TASK_FINISHED) .setData(ByteString.copyFromUtf8("eval_not_run")) // TODO[JIRA REEF-102]: a hack to pass closeEvaluator test, replace this with a better interface .setMessage("Evaluator Process exited with status " + String.valueOf(evaluatorProcessExitValue)) .build()); } // Shutdown Mesos Executor this.executorService.shutdown(); this.mesosExecutorDriver.stop(); }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.scheduler.Protos.Call.Reconcile.Task)) { return super.equals(obj); } org.apache.mesos.scheduler.Protos.Call.Reconcile.Task other = (org.apache.mesos.scheduler.Protos.Call.Reconcile.Task) obj; boolean result = true; result = result && (hasTaskId() == other.hasTaskId()); if (hasTaskId()) { result = result && getTaskId() .equals(other.getTaskId()); } result = result && (hasSlaveId() == other.hasSlaveId()); if (hasSlaveId()) { result = result && getSlaveId() .equals(other.getSlaveId()); } result = result && unknownFields.equals(other.unknownFields); return result; }
@Override public void run() { // performing "explicit" reconciliation; master will respond with the latest state for all logviewer tasks // in the framework scheduler's statusUpdate() method List<TaskStatus> taskStatuses = new ArrayList<TaskStatus>(); List<String> logviewerPaths = _zkClient.getChildren(_logviewerZkDir); if (logviewerPaths == null) { _driver.reconcileTasks(taskStatuses); return; } for (String path : logviewerPaths) { TaskID logviewerTaskId = TaskID.newBuilder() .setValue(new String(_zkClient.getNodeData(String.format("%s/%s", _logviewerZkDir, path)))) .build(); TaskStatus logviewerTaskStatus = TaskStatus.newBuilder() .setTaskId(logviewerTaskId) .setState(TaskState.TASK_RUNNING) .build(); taskStatuses.add(logviewerTaskStatus); } _driver.reconcileTasks(taskStatuses); LOG.info("Performing task reconciliation between scheduler and master on following tasks: {}", taskStatusListToTaskIDsString(taskStatuses)); } }, 0, TASK_RECONCILIATION_INTERVAL); // reconciliation performed every 5 minutes
/** * <code>required .mesos.TaskID task_id = 2;</code> */ public Builder mergeTaskId(org.apache.mesos.Protos.TaskID value) { if (taskIdBuilder_ == null) { if (((bitField0_ & 0x00000002) == 0x00000002) && taskId_ != null && taskId_ != org.apache.mesos.Protos.TaskID.getDefaultInstance()) { taskId_ = org.apache.mesos.Protos.TaskID.newBuilder(taskId_).mergeFrom(value).buildPartial(); } else { taskId_ = value; } onChanged(); } else { taskIdBuilder_.mergeFrom(value); } bitField0_ |= 0x00000002; return this; } /**
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.executor.Protos.Event.Acknowledged)) { return super.equals(obj); } org.apache.mesos.executor.Protos.Event.Acknowledged other = (org.apache.mesos.executor.Protos.Event.Acknowledged) obj; boolean result = true; result = result && (hasTaskId() == other.hasTaskId()); if (hasTaskId()) { result = result && getTaskId() .equals(other.getTaskId()); } result = result && (hasUuid() == other.hasUuid()); if (hasUuid()) { result = result && getUuid() .equals(other.getUuid()); } result = result && unknownFields.equals(other.unknownFields); return result; }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.executor.Protos.Event.Kill)) { return super.equals(obj); } org.apache.mesos.executor.Protos.Event.Kill other = (org.apache.mesos.executor.Protos.Event.Kill) obj; boolean result = true; result = result && (hasTaskId() == other.hasTaskId()); if (hasTaskId()) { result = result && getTaskId() .equals(other.getTaskId()); } result = result && (hasKillPolicy() == other.hasKillPolicy()); if (hasKillPolicy()) { result = result && getKillPolicy() .equals(other.getKillPolicy()); } result = result && unknownFields.equals(other.unknownFields); return result; }
/** * <code>required .mesos.TaskID task_id = 1;</code> */ public Builder mergeTaskId(org.apache.mesos.Protos.TaskID value) { if (taskIdBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && taskId_ != null && taskId_ != org.apache.mesos.Protos.TaskID.getDefaultInstance()) { taskId_ = org.apache.mesos.Protos.TaskID.newBuilder(taskId_).mergeFrom(value).buildPartial(); } else { taskId_ = value; } onChanged(); } else { taskIdBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
@Override public Protos.TaskInfo create(String taskId, Protos.Offer offer, List<Protos.Resource> resources, ExecutionParameters executionParameters) { logger.info("Creating task with taskId=" + taskId + " from offerId=" + offer.getId().getValue()); return Protos.TaskInfo.newBuilder() .setName(applicationName + ".task") .setSlaveId(offer.getSlaveId()) .setTaskId(Protos.TaskID.newBuilder().setValue(taskId)) .addAllResources(resources) .setContainer(Protos.ContainerInfo.newBuilder() .setType(Protos.ContainerInfo.Type.DOCKER) .setDocker(Protos.ContainerInfo.DockerInfo.newBuilder() .setImage(dockerImage) .addAllPortMappings(portMappings(executionParameters.getPortMappings())) .setNetwork(Protos.ContainerInfo.DockerInfo.Network.valueOf(networkMode)) ) .addAllVolumes(volumeMappings(executionParameters.getVolumeMappings())) ) .setCommand(command(executionParameters.getEnvironmentVariables())) .build(); }
/** * <code>required .mesos.TaskID task_id = 1;</code> */ public Builder mergeTaskId(org.apache.mesos.Protos.TaskID value) { if (taskIdBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && taskId_ != null && taskId_ != org.apache.mesos.Protos.TaskID.getDefaultInstance()) { taskId_ = org.apache.mesos.Protos.TaskID.newBuilder(taskId_).mergeFrom(value).buildPartial(); } else { taskId_ = value; } onChanged(); } else { taskIdBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
/** * <code>required .mesos.TaskID task_id = 1;</code> */ public Builder mergeTaskId(org.apache.mesos.Protos.TaskID value) { if (taskIdBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && taskId_ != null && taskId_ != org.apache.mesos.Protos.TaskID.getDefaultInstance()) { taskId_ = org.apache.mesos.Protos.TaskID.newBuilder(taskId_).mergeFrom(value).buildPartial(); } else { taskId_ = value; } onChanged(); } else { taskIdBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
/** * <code>required .mesos.TaskID task_id = 1;</code> */ public Builder mergeTaskId(org.apache.mesos.Protos.TaskID value) { if (taskIdBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && taskId_ != null && taskId_ != org.apache.mesos.Protos.TaskID.getDefaultInstance()) { taskId_ = org.apache.mesos.Protos.TaskID.newBuilder(taskId_).mergeFrom(value).buildPartial(); } else { taskId_ = value; } onChanged(); } else { taskIdBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
@Override public void send(ClusterState state, SchedulerDriver mockDriver, Scheduler scheduler) { Protos.TaskID taskId = builder.taskId == null ? state.getTaskId(builder.taskName) : Protos.TaskID.newBuilder().setValue(builder.taskId).build(); Protos.TaskStatus.Builder taskStatusBuilder = Protos.TaskStatus.newBuilder() .setTaskId(taskId) .setState(builder.taskState) .setMessage("This is a test status"); if (builder.readinessCheckExitCode.isPresent()) { taskStatusBuilder .getCheckStatusBuilder() .getCommandBuilder() .setExitCode(builder.readinessCheckExitCode.get()); } scheduler.statusUpdate(mockDriver, taskStatusBuilder.build()); }
@Test public void canParseTaskCappedTimestamp() throws Exception { final ZonedDateTime nowUTC = ZonedDateTime.now(ZoneOffset.UTC); when(clock.nowUTC()).thenReturn(nowUTC); final Protos.TaskID taskId = Protos.TaskID.newBuilder().setValue("TaskID").build(); Properties data = new Properties(); data.put("hostname", "hostname"); data.put("ipAddress", "ip address"); data.put("startedAt", nowUTC.withZoneSameInstant(ZoneId.of("Europe/Paris")).toString()); StringWriter writer = new StringWriter(); data.list(new PrintWriter(writer)); final Protos.TaskInfo taskInfo = createTaskInfo(taskId, ByteString.copyFromUtf8(writer.getBuffer().toString())); final Protos.TaskStatus taskStatus = Protos.TaskStatus.newBuilder() .setTaskId(taskId) .setState(Protos.TaskState.TASK_STAGING) .build(); final Task task = TaskInfoFactory.parse(taskInfo, taskStatus, clock); assertEquals(nowUTC, task.getStartedAt()); }
/** * Checks if the given taskId already exists or just finished running. If it has, then refuse the offer. * @param taskId The task id * @return True if the task already exists, false otherwise */ @VisibleForTesting boolean isExistingTask(TaskID taskId) { // If the task has already been queued, don't launch it again if (results.containsKey(taskId)) { LOGGER.info("Task " + taskId.getValue() + " has already been launched, ignoring and refusing offer"); return true; } // If the task has already finished, then do not start it up again even if we are offered it if (finishedTasks.contains(taskId)) { LOGGER.info("Task " + taskId.getValue() + " has already finished. Ignoring and refusing offer"); return true; } return false; }