public static String describe(Procedure<?> proc) { Map<String, Object> description = new LinkedHashMap<>(); description.put("ID", proc.getProcId()); description.put("PARENT_ID", proc.getParentProcId()); description.put("STATE", proc.getState()); description.put("OWNER", proc.getOwner()); description.put("TYPE", proc.getProcName()); description.put("START_TIME", new Date(proc.getSubmittedTime())); description.put("LAST_UPDATE", new Date(proc.getLastUpdate())); if (proc.isFailed()) { description.put("ERRORS", MasterProcedureUtil.unwrapRemoteIOException(proc).getMessage()); } description.put("PARAMETERS", parametersToObject(proc)); return JRubyFormat.print(description); }
public static Throwable getExceptionCause(final Procedure<?> procInfo) { assert procInfo.isFailed(); Throwable cause = procInfo.getException().getCause(); return cause == null ? procInfo.getException() : cause; }
public static byte[] waitForProcedureToComplete( final ProcedureExecutor<MasterProcedureEnv> procExec, final Procedure<?> proc, final long timeout) throws IOException { waitFor(procExec.getEnvironment(), timeout, "pid=" + proc.getProcId(), new ProcedureSyncWait.Predicate<Boolean>() { @Override if (proc.hasException()) { throw MasterProcedureUtil.unwrapRemoteIOException(proc); } else { return proc.getResult();
@Override public int compare(Procedure lhs, Procedure rhs) { long cmp = lhs.getParentProcId() - rhs.getParentProcId(); cmp = cmp != 0 ? cmp : lhs.getProcId() - rhs.getProcId(); return cmp < 0 ? -1 : cmp > 0 ? 1 : 0; } });
/** * @param a the first procedure to be compared. * @param b the second procedure to be compared. * @return true if the two procedures have the same parent */ public static boolean haveSameParent(Procedure<?> a, Procedure<?> b) { return a.hasParent() && b.hasParent() && (a.getParentProcId() == b.getParentProcId()); } }
protected synchronized RemoteProcedureException getException() { if (subprocStack != null) { for (Procedure<TEnvironment> proc: subprocStack) { if (proc.hasException()) { return proc.getException(); } } } return null; }
proc.setProcId(proto.getProcId()); proc.setState(proto.getState()); proc.setSubmittedTime(proto.getSubmittedTime()); proc.setLastUpdate(proto.getLastUpdate()); proc.setParentProcId(proto.getParentId()); proc.setOwner(proto.getOwner()); proc.setTimeout(proto.getTimeout()); proc.setStackIndexes(proto.getStackIdList()); assert proc.getState() == ProcedureProtos.ProcedureState.FAILED || proc.getState() == ProcedureProtos.ProcedureState.ROLLEDBACK : "The procedure must be failed (waiting to rollback) or rolledback"; proc.setFailure(RemoteProcedureException.fromProto(proto.getException())); proc.setResult(proto.getResult().toByteArray()); proc.setNonceKey(new NonceKey(proto.getNonceGroup(), proto.getNonce())); proc.lockedWhenLoading(); proc.bypass(null); proc.deserializeStateData(serializer);
.setProcId(proc.getProcId()) .setState(proc.getState()) .setSubmittedTime(proc.getSubmittedTime()) .setLastUpdate(proc.getLastUpdate()); if (proc.hasParent()) { builder.setParentId(proc.getParentProcId()); if (proc.hasTimeout()) { builder.setTimeout(proc.getTimeout()); if (proc.hasOwner()) { builder.setOwner(proc.getOwner()); final int[] stackIds = proc.getStackIndexes(); if (stackIds != null) { for (int i = 0; i < stackIds.length; ++i) { if (proc.hasException()) { RemoteProcedureException exception = proc.getException(); builder.setException( RemoteProcedureException.toProto(exception.getSource(), exception.getCause())); final byte[] result = proc.getResult(); if (result != null) { builder.setResult(UnsafeByteOperations.unsafeWrap(result)); proc.serializeStateData(serializer);
for (Procedure<?> proc : procedures) { out.write("\n <tr>\n <td>"); out.print( proc.getProcId() ); out.write("</td>\n <td>"); out.print( proc.hasParent() ? proc.getParentProcId() : "" ); out.write("</td>\n <td>"); out.print( escapeXml(proc.getState().toString() + (proc.isBypass() ? "(Bypass)" : "")) ); out.write("</td>\n <td>"); out.print( proc.hasOwner() ? escapeXml(proc.getOwner()) : "" ); out.write("</td>\n <td>"); out.print( escapeXml(proc.getProcName()) ); out.write("</td>\n <td>"); out.print( new Date(proc.getSubmittedTime()) ); out.write("</td>\n <td>"); out.print( new Date(proc.getLastUpdate()) ); out.write("</td>\n <td>"); out.print( escapeXml(proc.isFailed() ? proc.getException().unwrapRemoteIOException().getMessage() : "") ); out.write("</td>\n <td>"); out.print( escapeXml(ProcedureDescriber.describeParameters(proc)) );
if (subproc == null) { String msg = "subproc[" + i + "] is null, aborting the procedure"; procedure.setFailure(new RemoteProcedureException(msg, new IllegalArgumentIOException(msg))); return null; assert subproc.getState() == ProcedureState.INITIALIZING : subproc; subproc.setParentProcId(procedure.getProcId()); subproc.setRootProcId(rootProcId); subproc.setProcId(nextProcId()); procStack.addSubProcedure(subproc); if (!procedure.isFailed()) { procedure.setChildrenLatch(subprocs.length); switch (procedure.getState()) { case RUNNABLE: procedure.setState(ProcedureState.WAITING); break; case WAITING_TIMEOUT:
sb.append(getProcId()); if (hasParent()) { sb.append(", ppid="); sb.append(getParentProcId()); toStringState(sb); if (hasException()) { sb.append(", exception=" + getException()); toStringClassDetails(sb);
IdLock.Entry lockEntry = procExecutionLock.tryLockEntry(procedure.getProcId(), lockWait); if (lockEntry == null && !override) { LOG.debug("Waited {} ms, but {} is still running, skipping bypass with force={}", if (procedure.isFinished()) { LOG.debug("{} is already finished, skipping bypass", procedure); return false; if (procedure.hasChildren()) { if (recursive) { LOG.info("Recursive bypass on children of pid={}", procedure.getProcId()); this.procedures.forEachValue(1 /*Single-threaded*/, v -> v.getParentProcId() == procedure.getProcId()? v: null, bypassProcedure(v.getProcId(), lockWait, override, recursive); } catch (IOException e) { LOG.warn("Recursive bypass of pid={}", v.getProcId(), e); if (procedure.hasParent() && procedure.getState() != ProcedureState.RUNNABLE && procedure.getState() != ProcedureState.WAITING && procedure.getState() != ProcedureState.WAITING_TIMEOUT) { LOG.debug("Bypassing procedures in RUNNABLE, WAITING and WAITING_TIMEOUT states " + "(with no parent), {}", while (current != null) { LOG.debug("Bypassing {}", current); current.bypass(getEnvironment());
RemoteProcedureException exception = rootProc.getException(); rootProc.setFailure(exception); store.update(rootProc); if (!procExecutionLock.isHeldByCurrentThread(proc.getProcId())) { try { lockEntry = procExecutionLock.getLockEntry(proc.getProcId()); } catch (IOException e) { if (proc.isSuccess()) { if (!proc.isFinished() && proc.isYieldAfterExecutionStep(getEnvironment())) { return LockState.LOCK_YIELD_WAIT; StringUtils.humanTimeDiff(rootProc.elapsedTime())); procedureFinished(rootProc); return LockState.LOCK_ACQUIRED;
private void executeProcedure(Procedure<TEnvironment> proc) { if (proc.isFinished()) { LOG.debug("{} is already finished, skipping execution", proc); return; LOG.warn("RootProcedureState is null for " + proc.getProcId()); return; if (!proc.wasExecuted()) { switch (executeRollback(proc)) { case LOCK_ACQUIRED: assert proc.getState() == ProcedureState.RUNNABLE : proc; if (proc.isSuccess()) { proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true); LOG.info("Finished " + proc + " in " + StringUtils.humanTimeDiff(proc.elapsedTime())); if (proc.getProcId() == rootProcId) { procedureFinished(proc); } else {
@Test public void testAbortTimeoutWithChildren() { List<String> state = new ArrayList<>(); Procedure<Void> proc = new TestWaitingProcedure("wproc", state, true); proc.setTimeout(2500); long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); LOG.info(Objects.toString(state)); Procedure<?> result = procExecutor.getResult(rootId); assertTrue(state.toString(), result.isFailed()); ProcedureTestingUtility.assertIsTimeoutException(result); assertEquals(state.toString(), 3, state.size()); assertEquals("wproc-execute", state.get(0)); assertEquals("wproc-child-execute", state.get(1)); assertEquals("wproc-rollback", state.get(2)); } }
private void cleanupAfterRollbackOneStep(Procedure<TEnvironment> proc) { if (proc.removeStackIndex()) { if (!proc.isSuccess()) { proc.setState(ProcedureState.ROLLEDBACK); } // update metrics on finishing the procedure (fail) proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), false); if (proc.hasParent()) { store.delete(proc.getProcId()); procedures.remove(proc.getProcId()); } else { final long[] childProcIds = rollbackStack.get(proc.getProcId()).getSubprocedureIds(); if (childProcIds != null) { store.delete(proc, childProcIds); } else { store.update(proc); } } } else { store.update(proc); } }
public static Throwable assertProcFailed(final Procedure<?> result) { assertEquals(true, result.isFailed()); LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage()); return getExceptionCause(result); }
Procedure<?> result = executor.getResultOrProcedure(procId); if (result != null) { builder.setSubmittedTime(result.getSubmittedTime()); builder.setLastUpdate(result.getLastUpdate()); if (executor.isFinished(procId)) { builder.setState(GetProcedureResultResponse.State.FINISHED); if (result.isFailed()) { IOException exception = MasterProcedureUtil.unwrapRemoteIOException(result); builder.setException(ForeignExceptionUtil.toProtoForeignException(exception)); byte[] resultData = result.getResult(); if (resultData != null) { builder.setResult(UnsafeByteOperations.unsafeWrap(resultData));
final void restoreLock(TEnvironment env) { if (!lockedWhenLoading) { LOG.debug("{} didn't hold the lock before restarting, skip acquiring lock.", this); return; } if (isFinished()) { LOG.debug("{} is already finished, skip acquiring lock.", this); return; } if (isBypass()) { LOG.debug("{} is already bypassed, skip acquiring lock.", this); return; } // this can happen if the parent stores the sub procedures but before it can // release its lock, the master restarts if (getState() == ProcedureState.WAITING && !holdLock(env)) { LOG.debug("{} is in WAITING STATE, and holdLock=false, skip acquiring lock.", this); lockedWhenLoading = false; return; } LOG.debug("{} held the lock before restarting, call acquireLock to restore it.", this); LockState state = acquireLock(env); assert state == LockState.LOCK_ACQUIRED; }
@Override public boolean hasLockAccess(Procedure<?> proc) { if (exclusiveLockOwnerProcedure == null) { return false; } long lockOwnerId = exclusiveLockOwnerProcedure.getProcId(); if (proc.getProcId() == lockOwnerId) { return true; } if (!proc.hasParent()) { return false; } // fast path to check root procedure if (proc.getRootProcId() == lockOwnerId) { return true; } // check ancestors for (Procedure<?> p = proc;;) { if (p.getParentProcId() == lockOwnerId) { return true; } p = procedureRetriever.apply(p.getParentProcId()); if (p == null || !p.hasParent()) { return false; } } }