taskStatuses.add(taskStatusHolder.getTaskStatus().get()); } else { TaskStatus.Builder fakeTaskStatusBuilder = TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskStatusHolder.getTaskId().getId())) .setState(TaskState.TASK_STARTING);
.map(event -> event.getUpdate().getStatus()) .filter(status -> { if (!status.hasAgentId() || !status.getAgentId().hasValue()) { LOG.warn("Filtering out status update without agentId {}", status); return false;
private void unsafeProcessStatusUpdate(Protos.TaskStatus status, SingularityTaskId taskIdObj) { final String taskId = status.getTaskId().getValue(); if (status.hasTimestamp()) { timestamp = (long) (status.getTimestamp() * 1000); long delta = now - timestamp; LOG.debug("Update: task {} is now {} ({}) at {} (delta: {})", taskId, status.getState(), status.getMessage(), timestamp, JavaUtils.durationFromMillis(delta)); statusUpdateDeltas.put(now, delta); final ExtendedTaskState taskState = MesosUtils.fromTaskState(status.getState()); if (status.getState() == TaskState.TASK_LOST) { boolean isMesosFailure = status.getReason() == Reason.REASON_INVALID_OFFERS || status.getReason() == Reason.REASON_AGENT_REMOVED || status.getReason() == Reason.REASON_AGENT_RESTARTED || status.getReason() == Reason.REASON_AGENT_UNKNOWN || status.getReason() == Reason.REASON_MASTER_DISCONNECTED || status.getReason() == Reason.REASON_AGENT_DISCONNECTED; taskLostReasons.add(status.getReason()); new SingularityTaskHistoryUpdate(taskIdObj, timestamp, taskState, statusMessage, status.hasReason() ? Optional.of(status.getReason().name()) : Optional.<String>absent()); final SingularityCreateResult taskHistoryUpdateCreateResult = taskManager.saveTaskHistoryUpdate(taskUpdate);
.map(event -> event.getUpdate().getStatus()) .filter(status -> { if (!status.hasAgentId() || !status.getAgentId().hasValue()) { LOG.warn("Filtering out status update without agentId {}", status); return false;
private void unsafeProcessStatusUpdate(Protos.TaskStatus status, SingularityTaskId taskIdObj) { final String taskId = status.getTaskId().getValue(); if (status.hasTimestamp()) { timestamp = (long) (status.getTimestamp() * 1000); long delta = now - timestamp; LOG.debug("Update: task {} is now {} ({}) at {} (delta: {})", taskId, status.getState(), status.getMessage(), timestamp, JavaUtils.durationFromMillis(delta)); statusUpdateDeltas.put(now, delta); final ExtendedTaskState taskState = MesosUtils.fromTaskState(status.getState()); if (status.getState() == TaskState.TASK_LOST) { boolean isMesosFailure = status.getReason() == Reason.REASON_INVALID_OFFERS || status.getReason() == Reason.REASON_AGENT_REMOVED || status.getReason() == Reason.REASON_AGENT_RESTARTED || status.getReason() == Reason.REASON_AGENT_UNKNOWN || status.getReason() == Reason.REASON_MASTER_DISCONNECTED || status.getReason() == Reason.REASON_AGENT_DISCONNECTED; taskLostReasons.add(status.getReason()); new SingularityTaskHistoryUpdate(taskIdObj, timestamp, taskState, statusMessage, status.hasReason() ? Optional.of(status.getReason().name()) : Optional.<String>absent()); final SingularityCreateResult taskHistoryUpdateCreateResult = taskManager.saveTaskHistoryUpdate(taskUpdate);
.filter(event -> event.getType() == Event.Type.UPDATE && event.getUpdate().getStatus().hasUuid()) .zipWith(stateObservable, Tuple2::create) .doOnNext((Tuple2<Event, State<FrameworkID, TaskID, TaskState>> t) -> { final State<FrameworkID, TaskID, TaskState> state = t._2; final TaskStatus status = event.getUpdate().getStatus(); state.put(status.getTaskId(), status.getState()); }) .map((Tuple2<Event, State<FrameworkID, TaskID, TaskState>> t) -> { final TaskStatus status = t._1.getUpdate().getStatus(); return SchedulerCalls.ackUpdate(t._2.getFwId(), status.getUuid(), status.getAgentId(), status.getTaskId()); }) .map(SinkOperations::create) .filter(event -> event.getType() == Event.Type.ERROR || (event.getType() == Event.Type.UPDATE && event.getUpdate().getStatus().getState() == TaskState.TASK_ERROR)) .doOnNext(e -> LOGGER.warn("Task Error: {}", ProtoUtils.protoToString(e))) .map(e -> Optional.empty());
@Nullable final ByteString uuid ) { final org.apache.mesos.v1.Protos.TaskStatus.Builder builder = org.apache.mesos.v1.Protos.TaskStatus.newBuilder() .setAgentId(agentId) .setExecutorId(executorId)
@Nullable final ByteString uuid ) { final org.apache.mesos.v1.Protos.TaskStatus.Builder builder = org.apache.mesos.v1.Protos.TaskStatus.newBuilder() .setAgentId(agentId) .setExecutorId(executorId)
taskStatuses.add(taskStatusHolder.getTaskStatus().get()); } else { TaskStatus.Builder fakeTaskStatusBuilder = TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskStatusHolder.getTaskId().getId())) .setState(TaskState.TASK_STARTING);
private void runTest(RequestType requestType, Reason reason, boolean shouldRetry) { initRequestWithType(requestType, false); initFirstDeploy(); SingularityTask task = startTask(firstDeploy); Assert.assertEquals(taskManager.getPendingTaskIds().size(), 0); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); try { updateHandler.processStatusUpdateAsync(TaskStatus.newBuilder() .setState(TaskState.TASK_LOST) .setReason(reason) .setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())) .build()).get(); } catch (InterruptedException | ExecutionException e) { Assert.assertTrue(false); } if (shouldRetry) { Assert.assertEquals(requestManager.getPendingRequests().size(), 1); Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.RETRY); } else { if (requestManager.getPendingRequests().size() > 0) { Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.TASK_DONE); } } scheduler.drainPendingQueue(); } }
/** * <code>required .mesos.v1.TaskStatus status = 1;</code> */ public Builder mergeStatus(org.apache.mesos.v1.Protos.TaskStatus value) { if (statusBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && status_ != null && status_ != org.apache.mesos.v1.Protos.TaskStatus.getDefaultInstance()) { status_ = org.apache.mesos.v1.Protos.TaskStatus.newBuilder(status_).mergeFrom(value).buildPartial(); } else { status_ = value; } onChanged(); } else { statusBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
/** * <code>required .mesos.v1.TaskStatus status = 1;</code> */ public Builder mergeStatus(org.apache.mesos.v1.Protos.TaskStatus value) { if (statusBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && status_ != null && status_ != org.apache.mesos.v1.Protos.TaskStatus.getDefaultInstance()) { status_ = org.apache.mesos.v1.Protos.TaskStatus.newBuilder(status_).mergeFrom(value).buildPartial(); } else { status_ = value; } onChanged(); } else { statusBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
@Override public void applyMigration() { final long start = System.currentTimeMillis(); final List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds(); for (SingularityTaskId taskId : taskIds) { List<SingularityTaskHistoryUpdate> updates = Lists.reverse(taskManager.getTaskHistoryUpdates(taskId)); Optional<MesosTaskStatusObject> taskStatus = Optional.absent(); for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState().toTaskState().isPresent()) { Optional<SingularityTask> task = taskManager.getTask(taskId); taskStatus = Optional.of(mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.getId())) .setAgentId(MesosProtosUtils.toAgentId(task.get().getAgentId())) .setState(MesosProtosUtils.toTaskState(update.getTaskState())) .build())); break; } } SingularityTaskStatusHolder taskStatusHolder = new SingularityTaskStatusHolder(taskId, taskStatus, start, serverId, Optional.absent()); taskManager.saveLastActiveTaskStatus(taskStatusHolder); } }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.v1.scheduler.Protos.Event.Update)) { return super.equals(obj); } org.apache.mesos.v1.scheduler.Protos.Event.Update other = (org.apache.mesos.v1.scheduler.Protos.Event.Update) obj; boolean result = true; result = result && (hasStatus() == other.hasStatus()); if (hasStatus()) { result = result && getStatus() .equals(other.getStatus()); } result = result && unknownFields.equals(other.unknownFields); return result; }
@Test public void testTaskOddities() { // test unparseable status update TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue("task")) .setAgentId(AgentID.newBuilder().setValue("slave1")) .setState(TaskState.TASK_RUNNING); // should not throw exception: sms.statusUpdate(bldr.build()).join(); initRequest(); initFirstDeploy(); SingularityTask taskOne = launchTask(request, firstDeploy, 1, TaskState.TASK_STARTING); taskManager.deleteTaskHistory(taskOne.getTaskId()); Assert.assertTrue(taskManager.isActiveTask(taskOne.getTaskId().getId())); statusUpdate(taskOne, TaskState.TASK_RUNNING); statusUpdate(taskOne, TaskState.TASK_FAILED); Assert.assertTrue(!taskManager.isActiveTask(taskOne.getTaskId().getId())); Assert.assertEquals(2, taskManager.getTaskHistoryUpdates(taskOne.getTaskId()).size()); }
public CompletableFuture<Boolean> processStatusUpdateAsync(Protos.TaskStatus status) { return statusUpdatesSemaphore.call(() -> CompletableFuture.supplyAsync(() -> { final String taskId = status.getTaskId().getValue(); final Optional<SingularityTaskId> maybeTaskId = getTaskId(taskId); if (!maybeTaskId.isPresent()) { return false; } schedulerLock.runWithRequestLock( () -> unsafeProcessStatusUpdate(status, maybeTaskId.get()), maybeTaskId.get().getRequestId(), getClass().getSimpleName() ); return true; }, statusUpdatesExecutor) ); } }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.mesos.v1.executor.Protos.Call.Update)) { return super.equals(obj); } org.apache.mesos.v1.executor.Protos.Call.Update other = (org.apache.mesos.v1.executor.Protos.Call.Update) obj; boolean result = true; result = result && (hasStatus() == other.hasStatus()); if (hasStatus()) { result = result && getStatus() .equals(other.getStatus()); } result = result && unknownFields.equals(other.unknownFields); return result; }
protected MesosTaskStatusObject buildTaskStatus(SingularityTask task) { return mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder().setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())).setState(TaskState.TASK_RUNNING).build()); }
@POST @Path("/scheduler/statusUpdate/{taskId}/{taskState}") @Operation( summary = "Force an update for a specific task", responses = { @ApiResponse(responseCode = "403", description = "Test resource calls are currently not enabled, set `allowTestResourceCalls` to `true` in config yaml to enable") } ) public void statusUpdate(@PathParam("taskId") String taskId, @PathParam("taskState") String taskState) { checkForbidden(configuration.isAllowTestResourceCalls(), "Test resource calls are disabled (set isAllowTestResourceCalls to true in configuration)"); scheduler.statusUpdate(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId)) .setState(TaskState.valueOf(taskState)) .build()).join(); }
private CompletableFuture<Boolean> handleStatusUpdateAsync(TaskStatus status) { long start = System.currentTimeMillis(); return statusUpdateHandler.processStatusUpdateAsync(status) .whenCompleteAsync((result, throwable) -> { if (throwable != null) { LOG.error("Scheduler threw an uncaught exception processing status updates", throwable); notifyStopping(); abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(throwable)); } if (status.hasUuid()) { mesosSchedulerClient.acknowledge(status.getAgentId(), status.getTaskId(), status.getUuid()); } LOG.debug("Handled status update for {} in {}", status.getTaskId().getValue(), JavaUtils.duration(start)); }); } }