private boolean isTaskExpired(Closure closure, ClosureDescription taskDef) { if (closure.state != TaskStage.STARTED) { return false; } if (taskDef.resources == null) { logWarning("No constraints constraints bound to closure."); } if (closure.lastLeasedTimeMillis == null) { return false; } long timeElapsed = System.currentTimeMillis() - closure.lastLeasedTimeMillis; if (timeElapsed > (taskDef.resources.timeoutSeconds * 1000)) { logInfo("Timeout elapsed=%s, timeout=%s of closure=%s", timeElapsed, taskDef.resources.timeoutSeconds * 1000, closure.documentSelfLink); return true; } return false; }
private void completeCancelTask(ClosureDescription closureDesc, Closure closure) { String errorMsg = "Configured timeout of [%s] seconds has expired. Closure %s is" + " cancelled."; logInfo(errorMsg, closureDesc.resources.timeoutSeconds, closure.documentSelfLink); closure.state = TaskStage.CANCELLED; closure.errorMsg = String.format(errorMsg, closureDesc.resources.timeoutSeconds, closure.documentSelfLink); closure.endTimeMillis = System.currentTimeMillis(); sendSelfPatch(closure); getExecutionDriver(closureDesc) .cleanClosure(closure, (error) -> logWarning("Unable to clean resources for %s", closure.documentSelfLink)); }
private void processMaintenance(Operation op, Closure closure, ClosureDescription taskDef) { if (closure == null) { logWarning("Skip maintenance call..."); return; } if (isTaskExpired(closure, taskDef)) { completeCancelTask(taskDef, closure); } op.complete(); }
private void processExecution(Operation op, Closure closure, ClosureDescription taskDef) { switch (closure.state) { case CREATED: case CANCELLED: handleReadyState(op, closure, taskDef); break; case STARTED: handledLeasedState(op, closure); break; case FINISHED: case FAILED: handleDoneState(op, closure); break; default: logWarning("Unsupported closure lease state: %s", closure.state); } }
@Override public void handlePeriodicMaintenance(Operation post) { if (getProcessingStage() != ProcessingStage.AVAILABLE) { logWarning("Skipping maintenance since service is not available: %s ", getUri()); return; } sendRequest(Operation .createGet(getUri()) .setCompletion((op, ex) -> { if (ex != null) { logWarning("Failed to fetch closure state. Reason: %s", ex.getMessage()); post.fail(new Exception("Unable to fetch closure state.")); } else { Closure closure = op.getBody(Closure.class); handleMaintenance(post, closure); } })); }
private void fetchLogs(Closure closure, Runnable operation) { if (closure.resourceLinks == null || closure.resourceLinks.size() <= 0) { return; } String resourceLink = closure.resourceLinks.iterator().next(); String containerId = UriUtils.getLastPathSegment(resourceLink); String logsURI = ManagementUriParts.CONTAINER_LOGS + "?id=" + containerId; sendRequest(Operation .createGet(this, logsURI) .setCompletion((op, ex) -> { if (ex != null) { logWarning("Failed to fetch logs for closure! %s Reason: %s", closure.documentSelfLink, ex.getMessage()); } else { logInfo("Logs fetched successfully for closure: %s", closure.documentSelfLink); LogServiceState logState = op.getBody(LogServiceState.class); byte[] fetchedLogs = shrinkToMaxAllowedSize(logState.logs); if (shouldUpdateLogs(closure.logs, fetchedLogs)) { closure.logs = fetchedLogs; sendSelfPatch(closure); } operation.run(); } })); }
private void initializeTask(Operation post, Closure closure) { sendRequest(Operation .createGet(this, closure.descriptionLink) .setCompletion((op, ex) -> { if (ex != null) { logWarning("Failed to fetch closure definition closure! Reason: %s", ex.getMessage()); post.fail(new Exception("Unable to fetch script source.")); } else { initTask(closure, op); this.setState(post, closure); post.setBody(closure).complete(); } })); }
private void handleMaintenance(Operation post, Closure closure) { sendRequest(Operation .createGet(this, closure.descriptionLink) .setCompletion((o, e) -> { if (e != null) { logWarning("Failed to fetch closure definition. Reason: %s", Utils.toString(e)); sendRequest(Operation .createDelete(getUri()) .setCompletion((dop, dex) -> { if (dex != null) { logWarning("Self delete failed: %s", Utils.toString(dex)); } })); post.fail(new Exception("Unable to fetch closure definition: " + e .getMessage())); } else { ClosureDescription taskDef = o.getBody(ClosureDescription.class); processMaintenance(post, closure, taskDef); } })); }
protected void sendSelfPatch(Closure body) { sendRequest(Operation .createPatch(getUri()) .setBody(body) .setCompletion((o, ex) -> { if (ex != null) { logWarning("Self patch failed: %s", Utils.toString(ex)); } })); }
@Override public void handlePut(Operation put) { Closure reqClosure = put.getBody(Closure.class); Closure closure = this.getState(put); sendRequest(Operation .createGet(this, closure.documentSelfLink) .setCompletion((op, ex) -> { if (ex != null) { logWarning("Failed to execute closure! Reason: %s", ex.getMessage()); put.fail(new Exception("Unable to fetch closure state.")); } else { Closure currentState = op.getBody(Closure.class); if (reqClosure.inputs != null) { currentState.inputs.putAll(reqClosure.inputs); } this.setState(put, currentState); put.setBody(currentState).complete(); } })); }
private void handleReadyState(Operation op, Closure closure, ClosureDescription closureDesc) { ExecutionDriver execDriver = getExecutionDriver(closureDesc); if (execDriver == null) { logWarning("No exec driver found"); op.fail(new Exception("No execution driver available.")); return; } sendSelfPatch(closure); String token = getToken(op); execDriver.executeClosure(closure, closureDesc, token, (error) -> { if (error != null) { closure.state = TaskStage.FAILED; closure.errorMsg = error.getMessage(); sendSelfPatch(closure); } }); this.setState(op, closure); op.setBody(closure).complete(); }
.setCompletion((op, ex) -> { if (ex != null) { logWarning("Failed to execute closure! Reason: %s", ex.getMessage()); post.fail(new Exception("Unable to fetch script source.")); } else {
private void callWebhook(String webHookUriStr, Closure closure) { logInfo("Calling execution container for closure: %s", closure.documentSelfLink); URI webHookUri = UriUtils.buildUri(webHookUriStr); sendRequest(Operation .createPost(webHookUri) .setExpiration(Utils.fromNowMicrosUtc(TimeUnit.SECONDS .toMicros(ClosureProps.DEFAULT_WEB_HOOK_EXPIRATION_TIMEOUT))) .setBody(closure) .setCompletion((op, ex) -> { if (ex != null) { logWarning("Unable to send closure state to: %s. Reason: %s", webHookUriStr, Utils.toString(ex)); } else { logInfo("Successfully sent closure state to: %s", webHookUri); } })); }
private void createRequestTracker(ClosureTaskState state) { sendRequest(Operation.createPost(this, ManagementUriParts.REQUEST_STATUS) .setBodyNoCloning(fromTask(state)) .setCompletion((o, e) -> { if (e != null) { logWarning("Failed to create request tracker for: " + state.documentSelfLink, e); return; } state.requestTrackerLink = o.getBody(TaskStatusState.class).documentSelfLink; logInfo("Created request tracker: %s", state.requestTrackerLink); updateRequestTracker(state, ClosureProps.RETRIES_COUNT); })); }
.setCompletion((op, ex) -> { if (ex != null) { logWarning("Failed to fetch definition of closure: %s. Reason: %s", closure.documentSelfLink, Utils.toString(ex)); } else { closure.documentSelfLink); getExecutionDriver(closureDesc).cleanClosure(closure, (error) -> logWarning( "Unable to clean resources for %s", closure.documentSelfLink));
protected void updateRequestStatus(Closure closure) { ClosureTaskState state = fromClosure(closure); String requestStatusLink = ManagementUriParts.REQUEST_STATUS + "/" + getSelfId(); sendRequest(Operation .createGet(getHost(), requestStatusLink) .setCompletion((op, ex) -> { if (ex != null) { if (op.getStatusCode() == 404) { createRequestTracker(state); } else { logWarning("Unable to update request status for:" + closure .documentSelfLink, ex); } } else { state.requestTrackerLink = requestStatusLink; updateRequestTracker(state, ClosureProps.RETRIES_COUNT); } })); }
TimeUnit.MILLISECONDS); } else { logWarning("Failed to update request tracker: %s. Error: %s", state.requestTrackerLink, Utils.toString(ex));
protected void notifyCallerService(Closure closureState) { if (closureState.serviceTaskCallback.isEmpty()) { return; } if (closureState.serviceTaskCallback.isExternal()) { sendRequestStateToExternalUrl(closureState.serviceTaskCallback.serviceSelfLink, closureState); } else { ClosureCallbackCompleteResponse callbackResponse = new ClosureCallbackCompleteResponse(); if (closureState.state == TaskStage.FINISHED) { callbackResponse.copy(closureState.serviceTaskCallback.getFinishedResponse()); } else { callbackResponse .copy(closureState.serviceTaskCallback.getFailedResponse(new Exception( closureState.errorMsg))); } sendRequest( Operation.createPatch(this, closureState.serviceTaskCallback.serviceSelfLink) .setBody(callbackResponse) .setCompletion((o, e) -> { if (e != null) { logWarning("Notifying parent task %s failed: %s", o.getUri(), Utils.toString(e)); } })); } }
if (TaskState.isFailed(taskInfo) || TaskState.isCancelled(taskInfo)) { String errorMsg = "Failed to build runtime image! state: %s, Reason: %n%s"; logWarning(errorMsg, taskInfo.stage, taskInfo.failure.message); currentState.state = taskInfo.stage; currentState.errorMsg = String.format(errorMsg, taskInfo.stage,