public void logInterruptedException(final InterruptedException interruptedException) { LOG.warn("Worker interrupt signalled, not performing callback for task {} (message id: {})", taskMessage.getTaskId(), messageId, interruptedException); }
@Override public void messageReceived( TaskMessage taskMessage ) { LOGGER.info("New message received, ID: " +taskMessage.getTaskId()); try { messageWriter.writeMessage(taskMessage); LOGGER.info("Output task message with ID: "+taskMessage.getTaskId()); } catch ( IOException | CodecException e ) { LOGGER.error("Failure while handling received message, ID: "+taskMessage.getTaskId(), e); } }
/** * Discard the supplied task message. * * @param tm the task message to be discarded * @param queueMessageId the reference to the message this task arrived on */ public void discardTask(final TaskMessage tm, final String queueMessageId) throws TaskRejectedException { LOG.warn("Discarding task {} (message id: {})", tm.getTaskId(), queueMessageId); callback.discard(queueMessageId); }
public Delivery getDelivery(TaskMessage originatingTaskMessage) { final String expectedTaskId = originatingTaskMessage.getTaskId(); return getDelivery(expectedTaskId); }
public static void publishMessage(FileAndTaskMessage fileWithTaskMessage) throws CodecException, IOException { TaskMessage taskMessage = fileWithTaskMessage.getTaskMessage(); LOGGER.debug("Sending task message with task ID: "+taskMessage.getTaskId()); queueManager.publishMessage(taskMessage); LOGGER.debug("Sent task message with task ID: "+taskMessage.getTaskId()); }
private void validateTaskMessage(TaskMessage tm) throws InvalidTaskException { // The task message must be present so that the framework can // callback with a valid message final String taskId = tm.getTaskId(); if (taskId == null) { throw new InvalidTaskException("Task identifier not specified"); } }
/** * Creates a folder in the specified directory, using the ID of the provided task message. * @param message Task Message to pull ID from. * @param outputDirectory Directory to create new folder. * @return Path to the new folder. */ public String createFolderFromTaskMessageId(TaskMessage message, String outputDirectory){ String taskId = message.getTaskId(); File taskFolder = new File(outputDirectory, FileNameHelper.sanitizeForFilesystem(taskId)); if(!taskFolder.exists() || !taskFolder.isDirectory()){ taskFolder.mkdir(); } return taskFolder.getAbsolutePath(); }
@Override public void setResponse(final TaskRejectedException taskRejectedException) { incrementResponseCount(true); LOG.info("Worker requested to abandon task {} (message id: {})", taskMessage.getTaskId(), taskMessage, taskRejectedException); workerCallback.abandon(messageId, taskRejectedException); }
/** * Publishes a TaskMessage to the InputQueue * * @param message the message to publish * @throws CodecException if TaskMessage fails to serialize * @throws IOException if channel fails to publish the message */ public void publishMessage(TaskMessage message) throws CodecException, IOException { if(Strings.isNullOrEmpty(publishQueue)){ LOGGER.warn("Attempted to publish message when publish queue is not set. Message not published."); return; } LOGGER.info("Publishing task message " + message.getTaskId()); byte[] data = codec.serialise(message); publisherChannel.basicPublish("", publishQueue, MessageProperties.TEXT_PLAIN, data); }
protected Path getSaveFilePath(TestItem<TInput, TExpected> testItem, TaskMessage message) { String baseFileName = testItem.getTag() == null ? message.getTaskId() : testItem.getTag(); baseFileName = baseFileName + ".testcase"; Path filePath = Paths.get(outputFolder, baseFileName); return filePath; }
protected TaskMessage publishTaskAndAwaitResultAsTaskMessage(BlockingQueue<Event<QueuePublisher>> pubEvents, CountDownLatch latch, TaskMessage message, String expectedTaskMessage) throws InterruptedException, JsonProcessingException, CodecException { //add message to input queue //pubEvents.add(new PublishQueueEvent(PublishEventType.PUBLISH, 0L, mapper.writeValueAsBytes(message), INPUT_QUEUENAME)); pubEvents.add(new PublisherPublishEvent(mapper.writeValueAsBytes(message))); Delivery deliveredResult = null; final String awaitThisMessage = Strings.isNullOrEmpty(expectedTaskMessage) ? message.getTaskId() : expectedTaskMessage; //give some time for task to be processed and result consumed (result consumer should update latch on consumption) final boolean timeout = !latch.await(RESULT_TIMEOUTSECONDS, TimeUnit.SECONDS); return getResultAsTaskMessage(awaitThisMessage, timeout); }
/** * Checks whether a task is still active. If a status check cannot be performed then the task is assumed to be active. Checking * status may result in a change to the tracking info on the supplied task message. * * @param tm task message to be checked to verify whether the task is still active * @return true if the task is still active, false otherwise */ private boolean checkStatus(TaskMessage tm) throws InvalidJobTaskIdException { Objects.requireNonNull(tm); TrackingInfo tracking = tm.getTracking(); if (tracking != null) { Date statusCheckTime = tracking.getStatusCheckTime(); if (statusCheckTime == null || statusCheckTime.getTime() <= System.currentTimeMillis()) { return performJobStatusCheck(tm); } LOG.debug("Task {} active status is not being checked - it is not yet time for the status check to be performed: status check due at {}", tm.getTaskId(), statusCheckTime); } else { LOG.debug("Task {} active status is not being checked - the task message does not have tracking info", tm.getTaskId()); } //By default a task is considered to be active. return true; }
private void debugCurrentQueueItems() { logger.warn("Message queue currently contains: " + consumerImpl.getDelivery().size()); // do we have any items on the queue at all, if so, output what we have? for (Map.Entry<String, Delivery> deliveredItem : consumerImpl.getDelivery().entrySet()) { Delivery tmpDel = deliveredItem.getValue(); TaskMessage tm = getTaskMessageFromDelivery(tmpDel); logger.warn( String.format("Queue contains other item: {%s} TrackingInfo jobTaskId: {%s}", tm.getTaskId(), tm.getTracking() == null ? "null" : tm.getTracking().getJobTaskId())); } }
/** * Checks whether tracking of this task message should end when publishing to the specified queue. If tracking is to end then this * method removes and returns the tracking info from the task message. * * @param queueMsgId the reference to the message this task arrived on * @param queueToSend the queue to which the message is to be published * @param tm task message whose tracking info is to be checked * @return if tracking of the message terminates on publishing to the specified queue then the removed tracking info is returned; * otherwise null is returned and the tracking info is not removed from the message */ private TrackingInfo checkForTrackingTermination(final String queueMsgId, final String queueToSend, TaskMessage tm) { Objects.requireNonNull(queueMsgId); Objects.requireNonNull(tm); // queueToSend can be null for a dead end worker final TrackingInfo tracking = tm.getTracking(); if (tracking != null) { final String trackTo = tracking.getTrackTo(); if ((trackTo == null && queueToSend == null) || (trackTo != null && trackTo.equalsIgnoreCase(queueToSend))) { LOG.debug("Task {} (message id: {}): removing tracking info from this message as tracking ends on publishing to the queue {}.", tm.getTaskId(), queueMsgId, queueToSend); tm.setTracking(null); } } return tracking; } }
protected void checkTaskMessageReturnedTaskStatus(TaskMessage resultWrapper, String expectedTaskId, TaskStatus expectedStatus) { //check that this has the task ID we specified when adding to Input Queue Assert.assertEquals(expectedTaskId, resultWrapper.getTaskId()); TaskStatus status = resultWrapper.getTaskStatus(); //check that task status is x Assert.assertEquals(expectedStatus, status); }
@Override public void processDelivery(Delivery delivery) { System.out.print("New delivery"); try { TaskMessage taskMessage = codec.deserialise(delivery.getMessageData(), TaskMessage.class, DecodeMethod.LENIENT); System.out.println(taskMessage.getTaskId() + ", status: " + taskMessage.getTaskStatus()); synchronized (syncLock) { resultHandler.handleResult(taskMessage); } } catch (CodecException e) { e.printStackTrace(); throw new AssertionError("Failed: " + e.getMessage()); } catch (Exception e) { e.printStackTrace(); throw new AssertionError("Failed: " + e.getMessage()); } }
@Override public void send(final String queueMsgId, final TaskMessage responseMessage) { Objects.requireNonNull(queueMsgId); Objects.requireNonNull(responseMessage); LOG.debug("Sending task {} complete (message id: {})", responseMessage.getTaskId(), queueMsgId); final String queue = responseMessage.getTo(); checkForTrackingTermination(queueMsgId, queue, responseMessage); final byte[] output; try { output = codec.serialise(responseMessage); } catch (final CodecException ex) { throw new RuntimeException(ex); } final int priority = responseMessage.getPriority() == null ? 0 : responseMessage.getPriority(); try { workerQueue.publish("-1", output, queue, Collections.emptyMap(), priority); } catch (final QueueException ex) { throw new RuntimeException(ex); } }
@Override public void forward(String queueMsgId, String queue, TaskMessage forwardedMessage, Map<String, Object> headers) { Objects.requireNonNull(queueMsgId); Objects.requireNonNull(forwardedMessage); // queue can be null for a dead end worker LOG.debug("Task {} (message id: {}) being forwarded to queue {}", forwardedMessage.getTaskId(), queueMsgId, queue); checkForTrackingTermination(queueMsgId, queue, forwardedMessage); try { // If the queue is null, acknowledge the task rather than forwarding it if (queue == null) { workerQueue.acknowledgeTask(queueMsgId); } else { // Else forward the task byte[] output = codec.serialise(forwardedMessage); workerQueue.publish(queueMsgId, output, queue, headers, forwardedMessage.getPriority() == null ? 0 : forwardedMessage.getPriority()); stats.incrementTasksForwarded(); //TODO - I'm guessing this stat should not be updated for forwarded messages: // stats.getOutputSizes().update(output.length); } } catch (CodecException | QueueException e) { LOG.error("Cannot publish data for forwarded task {}, rejecting", forwardedMessage.getTaskId(), e); abandon(queueMsgId, e); } }
private void registerNewTaskImpl(final String queueMsgId, final byte[] taskMessage, Map<String, Object> headers) throws InvalidTaskException, TaskRejectedException { try { TaskMessage tm = codec.deserialise(taskMessage, TaskMessage.class, DecodeMethod.LENIENT); LOG.debug("Received task {} (message id: {})", tm.getTaskId(), queueMsgId); boolean poison = isTaskPoisoned(headers); validateTaskMessage(tm); boolean taskIsActive = checkStatus(tm); if (taskIsActive) { if (tm.getTo() != null && tm.getTo().equalsIgnoreCase(workerQueue.getInputQueue())) { LOG.debug("Task {} (message id: {}) on input queue {} {}", tm.getTaskId(), queueMsgId, workerQueue.getInputQueue(), (tm.getTo() != null) ? "is intended for this worker" : "has no explicit destination, therefore assuming it is intended for this worker"); executor.executeTask(tm, queueMsgId, poison, headers, codec); } else { LOG.debug("Task {} (message id: {}) is not intended for this worker: input queue {} does not match message destination queue {}", tm.getTaskId(), queueMsgId, workerQueue.getInputQueue(), tm.getTo()); executor.forwardTask(tm, queueMsgId, headers); } } else { LOG.debug("Task {} is no longer active. The task message (message id: {}) will not be executed", tm.getTaskId(), queueMsgId); executor.discardTask(tm, queueMsgId); } } catch (CodecException e) { throw new InvalidTaskException("Queue data did not deserialise to a TaskMessage", e); } catch (InvalidJobTaskIdException ijte) { throw new InvalidTaskException("TaskMessage contains an invalid job task identifier", ijte); } }
private TaskMessage createResponseMessage(final boolean includeTaskContext, final WorkerResponse response) { final Map<String, byte[]> responseContext = createFullResponseContext(includeTaskContext, response.getContext()); final String responseMessageType = response.getMessageType(); // Check if a tracking change is required. If empty string then no further changes required. final TrackingInfo trackingInfo; if ("".equals(response.getTrackTo())) { // No tracking changes required. trackingInfo = taskMessage.getTracking(); } else { // Set trackTo field to the specified value. trackingInfo = getTrackingInfoWithChanges(response.getTrackTo()); } final TaskMessage responseMessage = new TaskMessage( taskMessage.getTaskId(), responseMessageType, response.getApiVersion(), response.getData(), response.getTaskStatus(), responseContext, response.getQueueReference(), trackingInfo, new TaskSourceInfo(getWorkerName(responseMessageType), getWorkerVersion())); responseMessage.setPriority(priorityManager.getResponsePriority(taskMessage)); return responseMessage; }