private void fragmentDone(final FragmentStatus status) { final boolean stateChanged = updateFragmentStatus(status); if (stateChanged) { // since we're in the fragment done clause and this was a change from previous final NodeTracker node = nodeMap.get(status.getProfile().getEndpoint()); node.fragmentComplete(); finishedFragments.incrementAndGet(); } }
private boolean updateFragmentStatus(final FragmentStatus fragmentStatus) { final FragmentHandle fragmentHandle = fragmentStatus.getHandle(); final FragmentData data = fragmentDataMap.get(fragmentHandle); final FragmentState oldState = data.getState(); final boolean inTerminalState = isTerminal(oldState); final FragmentState currentState = fragmentStatus.getProfile().getState(); if (inTerminalState || (oldState == FragmentState.CANCELLATION_REQUESTED && !isTerminal(currentState))) { // Already in a terminal state, or invalid state transition from CANCELLATION_REQUESTED. This shouldn't happen. logger.warn(String.format("Received status message for fragment %s after fragment was in state %s. New state was %s", QueryIdHelper.getQueryIdentifier(fragmentHandle), oldState, currentState)); return false; } data.setStatus(fragmentStatus); return oldState != currentState; }
@Override public void fragmentStatusUpdate(FragmentStatus status) throws RpcException { ExternalId id = ExternalIdHelper.toExternal(status.getHandle().getQueryId()); ManagedForeman managed = externalIdToForeman.get(id); if (managed == null) { // TODO(DX-7242): this is a little chatty since a failed query will often log a bunch of fragments // We need a better mechanism to debug this. logger.info("A fragment status message arrived post query termination, dropping. Fragment [{}] reported a state of {}.", QueryIdHelper.getFragmentId(status.getHandle()), status.getProfile().getState()); } else { managed.foreman.updateStatus(status); } }
@Override public void statusUpdate(final FragmentStatus status) { logger.debug("New fragment status was provided to QueryManager of {}", status); switch(status.getProfile().getState()) { case AWAITING_ALLOCATION: case RUNNING: case CANCELLATION_REQUESTED: updateFragmentStatus(status); break; case FAILED: logger.info("Fragment {} failed, cancelling remaining fragments.", QueryIdHelper.getQueryIdentifier(status.getHandle())); completionListener.failed(UserRemoteException.create(status.getProfile().getError())); fragmentDone(status); break; case FINISHED: FragmentHandle fragmentHandle = status.getHandle(); if (fragmentHandle.getMajorFragmentId() == 0) { cancelExecutingFragments(); } fragmentDone(status); break; case CANCELLED: fragmentDone(status); break; default: throw new UnsupportedOperationException(String.format("Received status of %s", status)); } } };
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof com.dremio.exec.proto.CoordExecRPC.FragmentStatus)) { return super.equals(obj); } com.dremio.exec.proto.CoordExecRPC.FragmentStatus other = (com.dremio.exec.proto.CoordExecRPC.FragmentStatus) obj; boolean result = true; result = result && (hasProfile() == other.hasProfile()); if (hasProfile()) { result = result && getProfile() .equals(other.getProfile()); } result = result && (hasHandle() == other.hasHandle()); if (hasHandle()) { result = result && getHandle() .equals(other.getHandle()); } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; }
@java.lang.Override public int hashCode() { if (memoizedHashCode != 0) { return memoizedHashCode; } int hash = 41; hash = (19 * hash) + getDescriptorForType().hashCode(); if (hasProfile()) { hash = (37 * hash) + PROFILE_FIELD_NUMBER; hash = (53 * hash) + getProfile().hashCode(); } if (hasHandle()) { hash = (37 * hash) + HANDLE_FIELD_NUMBER; hash = (53 * hash) + getHandle().hashCode(); } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; }
@Override public FragmentInfo apply(final FragmentExecutor fragmentExecutor) { final FragmentStatus status = fragmentExecutor.getStatus(); final ExecProtos.FragmentHandle handle = fragmentExecutor.getHandle(); final MinorFragmentProfile profile = status == null ? null : status.getProfile(); Long memoryUsed = profile == null ? 0 : profile.getMemoryUsed(); Long rowsProcessed = profile == null ? 0 : getRowsProcessed(profile); Timestamp startTime = profile == null ? new Timestamp(0) : new Timestamp(profile.getStartTime()); return new FragmentInfo(dbContext.get().getEndpoint().getAddress(), QueryIdHelper.getQueryId(handle.getQueryId()), handle.getMajorFragmentId(), handle.getMinorFragmentId(), memoryUsed, rowsProcessed, startTime, fragmentExecutor.getBlockingStatus(), fragmentExecutor.getTaskDescriptor()); }
private boolean madeProgress(final FragmentStatus prev, final FragmentStatus cur) { final MinorFragmentProfile previous = prev.getProfile(); final MinorFragmentProfile current = cur.getProfile(); if (previous.getState() != current.getState()) { return true; } if (previous.getOperatorProfileCount() != current.getOperatorProfileCount()) { return true; } for(int i =0; i < current.getOperatorProfileCount(); i++){ if (madeProgress(previous.getOperatorProfile(i), current.getOperatorProfile(i))) { return true; } } return false; }
public FragmentState getState() { return status.getProfile().getState(); }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.CoordExecRPC.FragmentStatus message) throws java.io.IOException { if(message.hasProfile()) output.writeObject(1, message.getProfile(), com.dremio.exec.proto.SchemaUserBitShared.MinorFragmentProfile.WRITE, false); if(message.hasHandle()) output.writeObject(2, message.getHandle(), com.dremio.exec.proto.SchemaExecProtos.FragmentHandle.WRITE, false); } public boolean isInitialized(com.dremio.exec.proto.CoordExecRPC.FragmentStatus message)
public MinorFragmentProfile getProfile() { return status .getProfile() .toBuilder() .setLastUpdate(lastStatusUpdate) .setLastProgress(lastProgress) .build(); }
public Builder mergeFrom(com.dremio.exec.proto.CoordExecRPC.FragmentStatus other) { if (other == com.dremio.exec.proto.CoordExecRPC.FragmentStatus.getDefaultInstance()) return this; if (other.hasProfile()) { mergeProfile(other.getProfile()); } if (other.hasHandle()) { mergeHandle(other.getHandle()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }