/** * Returns a String representation of the time. If time is 0, then 'never' is returned. * <p> * This method is not particularly efficient since it generates a ton of litter. * * @param timeMillis time in millis * @return the String */ public static String timeToStringFriendly(long timeMillis) { return timeMillis == 0 ? "never" : timeToString(timeMillis); }
/** * Returns a String representation of the time. If time is 0, then 'never' is returned. * <p> * This method is not particularly efficient since it generates a ton of litter. * * @param timeMillis time in millis * @return the String */ public static String timeToStringFriendly(long timeMillis) { return timeMillis == 0 ? "never" : timeToString(timeMillis); }
@Override public String toString() { return "Invocation{" + "op=" + op + ", tryCount=" + tryCount + ", tryPauseMillis=" + tryPauseMillis + ", invokeCount=" + invokeCount + ", callTimeoutMillis=" + callTimeoutMillis + ", firstInvocationTimeMs=" + firstInvocationTimeMillis + ", firstInvocationTime='" + timeToString(firstInvocationTimeMillis) + '\'' + ", lastHeartbeatMillis=" + lastHeartbeatMillis + ", lastHeartbeatTime='" + timeToString(lastHeartbeatMillis) + '\'' + ", target=" + invTarget + ", pendingResponse={" + pendingResponse + '}' + ", backupsAcksExpected=" + backupsAcksExpected + ", backupsAcksReceived=" + backupsAcksReceived + ", connection=" + connection + '}'; }
@Override public String toString() { return "Invocation{" + "op=" + op + ", tryCount=" + tryCount + ", tryPauseMillis=" + tryPauseMillis + ", invokeCount=" + invokeCount + ", callTimeoutMillis=" + callTimeoutMillis + ", firstInvocationTimeMs=" + firstInvocationTimeMillis + ", firstInvocationTime='" + timeToString(firstInvocationTimeMillis) + '\'' + ", lastHeartbeatMillis=" + lastHeartbeatMillis + ", lastHeartbeatTime='" + timeToString(lastHeartbeatMillis) + '\'' + ", target=" + targetAddress + ", pendingResponse={" + pendingResponse + '}' + ", backupsAcksExpected=" + backupsAcksExpected + ", backupsAcksReceived=" + backupsAcksReceived + ", connection=" + connection + '}'; }
private void throwOperationTimeoutException(long startMillis, long nowInMillis, long elapsedMillis, Member lastFailedMember, Exception lastException) { throw new OperationTimeoutException("Registering listeners is timed out." + " Last failed member : " + lastFailedMember + ", " + " Current time: " + timeToString(nowInMillis) + ", " + " Start time : " + timeToString(startMillis) + ", " + " Client invocation timeout : " + invocationTimeoutMillis + " ms, " + " Elapsed time : " + elapsedMillis + " ms. ", lastException); } }
private static void throwOperationTimeoutException(long startMillis, long nowInMillis, long elapsedMillis, long invocationTimeoutMillis, Exception lastException) { throw new OperationTimeoutException("Creating cache config is timed out." + " Current time: " + timeToString(nowInMillis) + ", " + " Start time : " + timeToString(startMillis) + ", " + " Client invocation timeout : " + invocationTimeoutMillis + " ms, " + " Elapsed time : " + elapsedMillis + " ms. ", lastException); }
private static void appendHeartbeat(StringBuilder sb, long lastHeartbeatMillis) { if (lastHeartbeatMillis == 0) { sb.append("never. "); } else { sb.append(timeToString(lastHeartbeatMillis)).append(". "); } } }
private static void throwOperationTimeoutException(long startMillis, long nowInMillis, long elapsedMillis, long invocationTimeoutMillis, Exception lastException) { throw new OperationTimeoutException("Creating cache config is timed out." + " Current time: " + timeToString(nowInMillis) + ", " + " Start time : " + timeToString(startMillis) + ", " + " Client invocation timeout : " + invocationTimeoutMillis + " ms, " + " Elapsed time : " + elapsedMillis + " ms. ", lastException); }
private void throwOperationTimeoutException(long startMillis, long nowInMillis, long elapsedMillis, Member lastFailedMember, Exception lastException) { throw new OperationTimeoutException("Registering listeners is timed out." + " Last failed member : " + lastFailedMember + ", " + " Current time: " + timeToString(nowInMillis) + ", " + " Start time : " + timeToString(startMillis) + ", " + " Client invocation timeout : " + invocationTimeoutMillis + " ms, " + " Elapsed time : " + elapsedMillis + " ms. ", lastException); } }
private static void appendHeartbeat(StringBuilder sb, long lastHeartbeatMillis) { if (lastHeartbeatMillis == 0) { sb.append("never. "); } else { sb.append(timeToString(lastHeartbeatMillis)).append(". "); } }
private Exception newOperationTimeoutException(Throwable e, long invocationTimeoutMillis, long startTimeMillis) { StringBuilder sb = new StringBuilder(); sb.append("Creating transaction context timed out because exception occurred after client invocation timeout "); sb.append(invocationTimeoutMillis).append(" ms. "); sb.append("Current time: ").append(timeToString(currentTimeMillis())).append(". "); sb.append("Start time: ").append(timeToString(startTimeMillis)).append(". "); sb.append("Total elapsed time: ").append(currentTimeMillis() - startTimeMillis).append(" ms. "); String msg = sb.toString(); return new OperationTimeoutException(msg, e); }
private Object newOperationTimeoutException(Throwable e) { StringBuilder sb = new StringBuilder(); sb.append(this); sb.append(" timed out because exception occurred after client invocation timeout "); sb.append(invocationService.getInvocationTimeoutMillis()).append(" ms. "); sb.append("Current time: ").append(timeToString(currentTimeMillis())).append(". "); sb.append("Start time: ").append(timeToString(startTimeMillis)).append(". "); sb.append("Total elapsed time: ").append(currentTimeMillis() - startTimeMillis).append(" ms. "); String msg = sb.toString(); return new OperationTimeoutException(msg, e); }
@Override public String toString() { StringBuilder sb = new StringBuilder(getClass().getName()).append('{'); sb.append("serviceName='").append(getServiceName()).append('\''); sb.append(", identityHash=").append(System.identityHashCode(this)); sb.append(", partitionId=").append(partitionId); sb.append(", replicaIndex=").append(replicaIndex); sb.append(", callId=").append(callId); sb.append(", invocationTime=").append(invocationTime).append(" (").append(timeToString(invocationTime)).append(")"); sb.append(", waitTimeout=").append(waitTimeout); sb.append(", callTimeout=").append(callTimeout); toString(sb); sb.append('}'); return sb.toString(); } }
private Exception newOperationTimeoutException(Throwable e, long invocationTimeoutMillis, long startTimeMillis) { StringBuilder sb = new StringBuilder(); sb.append("Creating transaction context timed out because exception occurred after client invocation timeout "); sb.append(invocationTimeoutMillis).append(" ms. "); sb.append("Current time: ").append(timeToString(currentTimeMillis())).append(". "); sb.append("Start time: ").append(timeToString(startTimeMillis)).append(". "); sb.append("Total elapsed time: ").append(currentTimeMillis() - startTimeMillis).append(" ms. "); String msg = sb.toString(); return new OperationTimeoutException(msg, e); }
private Object newOperationTimeoutException(Throwable e) { StringBuilder sb = new StringBuilder(); sb.append(this); sb.append(" timed out because exception occurred after client invocation timeout "); sb.append(invocationService.getInvocationTimeoutMillis()).append(" ms. "); sb.append("Current time: ").append(timeToString(currentTimeMillis())).append(". "); sb.append("Start time: ").append(timeToString(startTimeMillis)).append(". "); sb.append("Total elapsed time: ").append(currentTimeMillis() - startTimeMillis).append(" ms. "); String msg = sb.toString(); return new OperationTimeoutException(msg, e); }
@Override public String toString() { StringBuilder sb = new StringBuilder(getClass().getName()).append('{'); sb.append("serviceName='").append(getServiceName()).append('\''); sb.append(", identityHash=").append(System.identityHashCode(this)); sb.append(", partitionId=").append(partitionId); sb.append(", replicaIndex=").append(replicaIndex); sb.append(", callId=").append(callId); sb.append(", invocationTime=").append(invocationTime).append(" (").append(timeToString(invocationTime)).append(")"); sb.append(", waitTimeout=").append(waitTimeout); sb.append(", callTimeout=").append(callTimeout); toString(sb); sb.append('}'); return sb.toString(); } }
private Object newOperationTimeoutException(boolean heartbeatTimeout) { StringBuilder sb = new StringBuilder(); if (heartbeatTimeout) { sb.append(invocation.op.getClass().getSimpleName()) .append(" invocation failed to complete due to operation-heartbeat-timeout. "); sb.append("Current time: ").append(timeToString(currentTimeMillis())).append(". "); sb.append("Start time: ").append(timeToString(invocation.firstInvocationTimeMillis)).append(". "); sb.append("Total elapsed time: ") .append(currentTimeMillis() - invocation.firstInvocationTimeMillis).append(" ms. "); long lastHeartbeatMillis = invocation.lastHeartbeatMillis; sb.append("Last operation heartbeat: "); appendHeartbeat(sb, lastHeartbeatMillis); long lastHeartbeatFromMemberMillis = invocation.context.invocationMonitor .getLastMemberHeartbeatMillis(invocation.getTargetAddress()); sb.append("Last operation heartbeat from member: "); appendHeartbeat(sb, lastHeartbeatFromMemberMillis); } else { sb.append(invocation.op.getClass().getSimpleName()) .append(" got rejected before execution due to not starting within the operation-call-timeout of: ") .append(invocation.callTimeoutMillis).append(" ms. "); sb.append("Current time: ").append(timeToString(currentTimeMillis())).append(". "); sb.append("Start time: ").append(timeToString(invocation.firstInvocationTimeMillis)).append(". "); sb.append("Total elapsed time: ") .append(currentTimeMillis() - invocation.firstInvocationTimeMillis).append(" ms. "); } sb.append(invocation); String msg = sb.toString(); return new ExecutionException(msg, new OperationTimeoutException(msg)); }
/** * Removes the {@code member} if it has not sent any heartbeats in {@link GroupProperty#MAX_NO_HEARTBEAT_SECONDS}. * If it has not sent any heartbeats in {@link #HEART_BEAT_INTERVAL_FACTOR} heartbeat intervals, it will log a warning. * * @param now the current cluster clock time * @param member the member which needs to be checked * @return if the member has been removed */ private boolean suspectMemberIfNotHeartBeating(long now, Member member) { if (clusterService.getMembershipManager().isMemberSuspected(member.getAddress())) { return true; } long lastHeartbeat = heartbeatFailureDetector.lastHeartbeat(member); if (!heartbeatFailureDetector.isAlive(member, now)) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); String reason = format("Suspecting %s because it has not sent any heartbeats since %s." + " Now: %s, heartbeat timeout: %d ms, suspicion level: %.2f", member, timeToString(lastHeartbeat), timeToString(now), maxNoHeartbeatMillis, suspicionLevel); logger.warning(reason); clusterService.suspectMember(member, reason, true); return true; } if (logger.isFineEnabled() && (now - lastHeartbeat) > heartbeatIntervalMillis * HEART_BEAT_INTERVAL_FACTOR) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); logger.fine(format("Not receiving any heartbeats from %s since %s, suspicion level: %.2f", member, timeToString(lastHeartbeat), suspicionLevel)); } return false; }
private void monitor(ClientEndpoint clientEndpoint) { // C++ client does not send heartbeat over its owner connection for versions before 3.10 // We are skipping checking heartbeat for cpp owner connection on those versions. if (clientEndpoint.isOwnerConnection() && ClientType.CPP.equals(clientEndpoint.getClientType()) && clientEndpoint.getClientVersion() < BuildInfo.calculateVersion("3.10")) { return; } Connection connection = clientEndpoint.getConnection(); long lastTimePacketReceived = connection.lastReadTimeMillis(); long timeoutInMillis = SECONDS.toMillis(heartbeatTimeoutSeconds); long currentTimeMillis = Clock.currentTimeMillis(); if (lastTimePacketReceived + timeoutInMillis < currentTimeMillis) { String message = "Client heartbeat is timed out, closing connection to " + connection + ". Now: " + timeToString(currentTimeMillis) + ". LastTimePacketReceived: " + timeToString(lastTimePacketReceived); connection.close(message, null); } } }
private void monitor(ClientEndpoint clientEndpoint) { // C++ client does not send heartbeat over its owner connection for versions before 3.10 // We are skipping checking heartbeat for cpp owner connection on those versions. if (clientEndpoint.isOwnerConnection() && ClientType.CPP.equals(clientEndpoint.getClientType()) && clientEndpoint.getClientVersion() < BuildInfo.calculateVersion("3.10")) { return; } Connection connection = clientEndpoint.getConnection(); long lastTimePacketReceived = connection.lastReadTimeMillis(); long timeoutInMillis = SECONDS.toMillis(heartbeatTimeoutSeconds); long currentTimeMillis = Clock.currentTimeMillis(); if (lastTimePacketReceived + timeoutInMillis < currentTimeMillis) { String message = "Client heartbeat is timed out, closing connection to " + connection + ". Now: " + timeToString(currentTimeMillis) + ". LastTimePacketReceived: " + timeToString(lastTimePacketReceived); connection.close(message, null); } } }