private ExecutionMessage createPendingExecutionMessage(Execution nextStepExecution) { //take care of worker group String groupName = nextStepExecution.getGroupName(); if (groupName == null) { groupName = WorkerNode.DEFAULT_WORKER_GROUPS[0]; } return new ExecutionMessage(ExecutionMessage.EMPTY_EXEC_STATE_ID, ExecutionMessage.EMPTY_WORKER, groupName, executionMessage.getMsgId(), ExecStatus.PENDING, converter.createPayload(nextStepExecution), 0).setWorkerKey(executionMessage.getWorkerKey()); }
private void ackMessages(List<ExecutionMessage> newMessages) throws InterruptedException { ExecutionMessage cloned; for (ExecutionMessage message : newMessages) { // create a unique id for this lane in this specific worker to be used in out buffer optimization //logger.error("ACK FOR MESSAGE: " + message.getMsgId() + " : " + message.getExecStateId()); message.setWorkerKey(message.getMsgId() + " : " + message.getExecStateId()); cloned = (ExecutionMessage) message.clone(); cloned.setStatus(ExecStatus.IN_PROGRESS); cloned.incMsgSeqId(); message.incMsgSeqId(); // increment the original message seq too in order to preserve the order of all messages of entire step cloned.setPayload(null); //payload is not needed in ack - make it null in order to minimize the data that is being sent outBuffer.put(cloned); } }
@Override public String toString(){ StringBuilder str = new StringBuilder(); boolean isAck = this.getExecutionObject() == null; str.append(" ExecutionId:").append(this.msgId). append(" ExecStateId:").append(this.execStateId). append(" Status:").append(this.status). append(" WorkerKey:").append(this.getId()). append(" IsAck:").append(isAck); return str.toString(); }
private ExecutionMessage createTerminatedExecutionMessage(Execution nextStepExecution) { Payload payload = converter.createPayload(nextStepExecution); //we need the payload ExecutionMessage finalMessage = (ExecutionMessage) executionMessage.clone(); finalMessage.setStatus(ExecStatus.TERMINATED); //in queue it is checked and finish flow is called finalMessage.incMsgSeqId(); finalMessage.setPayload(payload); return finalMessage; }
private void fillPayload(ExecutionMessage msg) { if (msg.getPayload() == null){ Map<Long, Payload> payloadMap = executionQueueService.readPayloadByExecutionIds(msg.getExecStateId()); Payload payload = payloadMap.get(msg.getExecStateId()); msg.setPayload(payload); } }
private boolean isExecutionPaused(Execution nextStepExecution) { //If execution was paused if(nextStepExecution == null){ //set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); //execution was paused - send only the FINISHED message! try { outBuffer.put(executionMessage); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", e); } return true; } else { return false; } }
if ( msg.getWorkerId().equals(ExecutionMessage.EMPTY_WORKER) && msg.getStatus() == ExecStatus.PENDING) { if (groupWorkersMap == null) { String engineVersionId = engineVersionService.getEngineVersionId(); String workerId = chooseWorker(msg.getWorkerGroup(), groupWorkersMap,randIntGenerator); if (workerId == null) { logger.warn("Can't assign worker for group name: " + msg.getWorkerGroup() + " , because there are no available workers for that group."); ExecutionMessage stepFinishMessage = (ExecutionMessage) msg.clone(); stepFinishMessage.setStatus(ExecStatus.FINISHED); stepFinishMessage.incMsgSeqId(); assignMessages.add(stepFinishMessage); ExecutionMessage flowFailedMessage = (ExecutionMessage) stepFinishMessage.clone(); flowFailedMessage.setStatus(ExecStatus.FAILED); addErrorMessage(flowFailedMessage); flowFailedMessage.incMsgSeqId(); assignMessages.add(flowFailedMessage); } else { msg.setStatus(ExecStatus.ASSIGNED); msg.incMsgSeqId(); msg.setWorkerId(workerId);
@Override public void run() { String executionId = executionMessage.getMsgId(); try { if(executionMessage.getExecutionObject() != null){ execution = executionMessage.getExecutionObject(); execution = converter.extractExecution(executionMessage.getPayload()); logger.error("Error during execution!!!", ex); executionMessage.setStatus(ExecStatus.FAILED); executionMessage.incMsgSeqId(); //new status must be with incremented msg_seq_id - otherwise will be recovered and we will get duplications if(executionMessage.getPayload() == null){ executionMessage.setPayload(converter.createPayload(execution)); //this is done since we could get here from InBuffer shortcut - so no payload... and for FAILED message we need to set the payload
executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setStepPersist(true); executionMessage.setStepPersistId(nextStepExecution.getSystemContext().getStepPersistId()); executionMessage.setPayload(converter.createPayload(nextStepExecution)); ExecutionMessage inProgressMessageForInBuffer = (ExecutionMessage) inProgressMessage.clone(); inProgressMessageForInBuffer.setPayload(null); //we do not need the payload for the inBuffer shortcut, we have execution there
@Override public void setValues(PreparedStatement ps, int i) throws SQLException { ExecutionMessage msg = messages.get(i); ps.setLong(1, idGen.next()); ps.setLong(2, msg.getExecStateId()); ps.setString(3, msg.getWorkerId()); ps.setString(4, msg.getWorkerGroup()); ps.setInt(5, msg.getStatus().getNumber()); ps.setInt(6, msg.getMsgSeqId()); ps.setLong(7, Calendar.getInstance().getTimeInMillis()); ps.setLong(8, version); }
@Override public ExecutionMessage mapRow(ResultSet rs, int rowNum) throws SQLException { return new ExecutionMessage(rs.getLong("EXEC_STATE_ID"), rs.getString("ASSIGNED_WORKER"), rs.getString("EXEC_GROUP"), "-1", ExecStatus.find(rs.getInt("STATUS")), null, rs.getInt("MSG_SEQ_ID"), rs.getLong("CREATE_TIME")); } }
@Override public void onEnqueue(List<ExecutionMessage> messages, int queueSize) { if (logger.isDebugEnabled()) { logger.debug("Enqueue " + messages.size() + " messages:"); logger.debug("queue size: " + queueSize); if (logger.isTraceEnabled()) { for (ExecutionMessage msg : messages) { String message = String.format( "Enqueue msgId= %s:%s,workerId=%s,status=%s", msg.getMsgUniqueId(), String.valueOf(msg.getMsgSeqId()), msg.getWorkerId(), msg.getStatus().toString() ); logger.trace(message); } } } }
@Override public void setValues(PreparedStatement ps, int i) throws SQLException { ExecutionMessage msg = stateMessages.get(i); ps.setLong(1, msg.getExecStateId()); ps.setString(2, msg.getMsgId()); ps.setBytes(3, msg.getPayload().getData()); }
if (msg.getExecStateId() == ExecutionMessage.EMPTY_EXEC_STATE_ID) { long execStateId = executionQueueRepository.generateExecStateId(); msg.setExecStateId(execStateId); stateMessages.add(msg); } else if (msg.getPayload() != null && msg.getStatus() == ExecStatus.IN_PROGRESS) { stateMessages.add(msg);
ExecutionMessage secondMessage = (ExecutionMessage) messages.get(1); List<Message> toPersistMessages = filerToPersistMessages(messages.subList(2, messages.size() - 1)); if (firstMessage.getStatus().equals(ExecStatus.IN_PROGRESS)) { resultAfterShrink.add(secondMessage); resultAfterShrink.addAll(toPersistMessages); if(secondMessage.isStepPersist()&& secondMessage.getStatus().equals(ExecStatus.FINISHED) ){ resultAfterShrink.add(secondMessage);
@Override @Transactional public void enqueueMessages(List<ExecutionMessage> messages, ExecStatus messageStatus) { if(!CollectionUtils.isEmpty(messages)){ for(ExecutionMessage msg:messages){ msg.setStatus(messageStatus); msg.setWorkerId(ExecutionMessage.EMPTY_WORKER); msg.incMsgSeqId(); } executionQueueService.enqueue(messages); } } }
private void addErrorMessage(ExecutionMessage message) { String group = message.getWorkerGroup(); Execution execution = converter.extractExecution(message.getPayload()); execution.getSystemContext().setNoWorkerInGroup(group); Payload payload = converter.createPayload(execution); message.setPayload(payload); }
public List<ExecutionMessage> pollMessagesWithoutAck(int maxSize, long minVersionAllowed) { pollMessagesWithoutAckJdbcTemplate.setStatementBatchSize(maxSize); try { String sqlStat = QUERY_MESSAGES_WITHOUT_ACK_SQL; Object[] values = {ExecStatus.SENT.getNumber(), minVersionAllowed}; long time = System.currentTimeMillis(); List<ExecutionMessage> result = pollMessagesWithoutAckJdbcTemplate.query(sqlStat, values, new ExecutionMessageWithoutPayloadRowMapper()); if (!result.isEmpty()) { logger.warn("Pool " + result.size() + " messages without ack, version = " + minVersionAllowed); if (logger.isDebugEnabled()) { for (ExecutionMessage msg : result) { logger.debug("Recovery msg [" + msg.getExecStateId() + "," + msg.getStatus() + "," + msg.getCreateDate() + "]"); } } } if (logger.isTraceEnabled()) logger.trace("Query [" + sqlStat + "] took " + (System.currentTimeMillis() - time) + " ms"); if (logger.isDebugEnabled()) { logger.debug("Got msg without ack :" + result.size() + ",for version:" + minVersionAllowed); } return result; } finally { pollMessagesWithoutAckJdbcTemplate.clearStatementBatchSize(); } }
protected List<Message> filerToPersistMessages(List<Message> messages){ List<Message> result = new ArrayList<>(); //We need to get from the list the FINISHED persisted messages for(Message msg : messages){ if(((ExecutionMessage)msg).isStepPersist() && ((ExecutionMessage)msg).getStatus().equals(ExecStatus.FINISHED)){ result.add(msg); } } return result; }
msg.setStatus(ExecStatus.SENT); msg.incMsgSeqId();