public boolean isTerminated(ContainerCreationStatus container) { Pod pod = client.pods().withName(container.getContainerName()).get(); if (pod == null) { logger.info("Container {} has no pod - terminal.", container); return true; } else { List<ContainerStatus> containerStatuses = pod.getStatus().getContainerStatuses(); Optional<ContainerStateTerminated> terminated = containerStatuses.stream() .flatMap(status -> Optional.ofNullable(status.getState()).map(Stream::of).orElse(Stream.empty())) .flatMap(state -> Optional.ofNullable(state.getTerminated()).map(Stream::of).orElse(Stream.empty())) .findFirst(); terminated.ifPresent(state -> logger.info("Container {} is {} - terminal.", container, state)); return terminated.isPresent(); } }
public ContainerStatusBuilder(ContainerStatus instance,Boolean validationEnabled){ this.fluent = this; this.withContainerID(instance.getContainerID()); this.withImage(instance.getImage()); this.withImageID(instance.getImageID()); this.withLastState(instance.getLastState()); this.withName(instance.getName()); this.withReady(instance.getReady()); this.withRestartCount(instance.getRestartCount()); this.withState(instance.getState()); this.validationEnabled = validationEnabled; }
private void waitUntilContainerIsReady() throws IOException { try { Pod pod = client.pods().inNamespace(namespace).withName(podName) .waitUntilReady(CONTAINER_READY_TIMEOUT, TimeUnit.MINUTES); if (pod == null || pod.getStatus() == null || pod.getStatus().getContainerStatuses() == null) { throw new IOException("Failed to execute shell script inside container " + "[" + containerName + "] of pod [" + podName + "]." + "Failed to get container status"); } for (ContainerStatus info : pod.getStatus().getContainerStatuses()) { if (info.getName().equals(containerName)) { if (info.getReady()) { return; } else { // container died in the meantime throw new IOException("container [" + containerName + "] of pod [" + podName + "] is not ready, state is " + info.getState()); } } } throw new IOException("container [" + containerName + "] does not exist in pod [" + podName + "]"); } catch (InterruptedException | KubernetesClientTimeoutException e) { throw new IOException("Failed to execute shell script inside container " + "[" + containerName + "] of pod [" + podName + "]." + " Timed out waiting for container to become ready!", e); } } };
@Override public boolean test(ContainerStatus containerStatus) { return containerStatus.getLastState() != null && containerStatus.getState() != null && containerStatus.getLastState().getTerminated() != null && containerStatus.getState().getWaiting() != null && containerStatus.getState().getWaiting().getReason() != null && containerStatus.getState().getWaiting().getReason().contains("CrashLoopBackOff"); } });
for (ContainerStatus containerStatus : containerStatuses) { if (restartCount == 0) { Integer restartCountValue = containerStatus.getRestartCount(); if (restartCountValue != null) { restartCount = restartCountValue.intValue(); ContainerState state = containerStatus.getState(); if (state != null) { ContainerStateWaiting waiting = state.getWaiting(); String containerName = containerStatus.getName(); if (waiting != null) { session.getLogger().warn("Waiting for container:" + containerName + ". Reason:" + waiting.getReason());
result.put("container.restartCount", "" + containerStatus.getRestartCount()); if (containerStatus.getLastState() != null && containerStatus.getLastState().getTerminated() != null) { result.put("container.lastState.terminated.exitCode", "" + containerStatus.getLastState().getTerminated().getExitCode()); result.put("container.lastState.terminated.reason", containerStatus.getLastState().getTerminated().getReason()); if (containerStatus.getState() != null && containerStatus.getState().getTerminated() != null) { result.put("container.state.terminated.exitCode", "" + containerStatus.getState().getTerminated().getExitCode()); result.put("container.state.terminated.reason", containerStatus.getState().getTerminated().getReason());
ContainerStatus status, Stats stats) { final ContainerStateTerminated terminated = status.getState().getTerminated(); if (terminated.getMessage() == null) { LOG.warn("Missing termination log message for workflow instance {} container {}", workflowInstance, status.getContainerID()); stats.recordTerminationLogMissing(); } else { LOG.warn("Exit code mismatch for workflow instance {} container {}. Container exit code: {}. " + "Termination log exit code: {}", workflowInstance, status.getContainerID(), terminated.getExitCode(), message.exitCode); stats.recordExitCodeMismatch(); stats.recordTerminationLogInvalid(); LOG.warn("Unexpected termination log message for workflow instance {} container {}", workflowInstance, status.getContainerID(), e); status.getContainerID()); return Optional.empty(); } else {
public HashMap<String, String> getLogsOfTerminatedPods(String namespace) { HashMap<String, String> terminatedPodsLogs = new HashMap<>(); try { client.pods().inNamespace(namespace).list().getItems().forEach(pod -> { pod.getStatus().getContainerStatuses().forEach(containerStatus -> { log.info("pod:'{}' : restart count '{}'", pod.getMetadata().getName(), containerStatus.getRestartCount()); if (containerStatus.getRestartCount() > 0) { terminatedPodsLogs.put( pod.getMetadata().getName(), client.pods().inNamespace(namespace) .withName(pod.getMetadata().getName()) .inContainer(containerStatus.getName()) .terminated().getLog()); } }); }); } catch (Exception allExceptions) { log.warn("Searching in terminated pods failed! No logs of terminated pods will be stored."); allExceptions.printStackTrace(); } return terminatedPodsLogs; }
/** * Returns the containers state, lazily creating any objects if required. */ public static ContainerState getOrCreateContainerState(Pod pod, String containerName) { ContainerStatus containerInfo = getOrCreateContainerInfo(pod, containerName); ContainerState state = containerInfo.getState(); if (state == null) { state = new ContainerState(); containerInfo.setState(state); } return state; }
/** * Returns the container information for the given pod and container name, lazily creating as required */ public static ContainerStatus getOrCreateContainerInfo(Pod pod, String containerName) { List<ContainerStatus> containerStatuses = getOrCreateContainerStatuses(pod); for (ContainerStatus containerStatus : containerStatuses) { String containerID = containerStatus.getContainerID(); if (Objects.equal(containerName, containerID)) { return containerStatus; } } ContainerStatus status = new ContainerStatus(); status.setContainerID(containerName); containerStatuses.add(status); return status; }
@Override public boolean test(ContainerStatus containerStatus) { return containerStatus.getRestartCount() == 0; } }, new ContainerStatusCondition("state is terminated") {
static Optional<ContainerStatus> getMainContainerStatus(Pod pod) { return readPodWorkflowInstance(pod) .flatMap(wfi -> pod.getStatus().getContainerStatuses().stream() .filter(status -> isMainContainer(status.getName(), pod)) .findAny()); }
public ContainerStatusBuilder(){ this(new ContainerStatus()); } public ContainerStatusBuilder( ContainerStatusFluent<?> fluent ){
@Override public boolean test(ContainerStatus containerStatus) { return containerStatus.getReady(); } });
public static String getDockerContainerID(Pod pod) { PodStatus status = pod.getStatus(); if (status != null) { List<ContainerStatus> containerStatuses = status.getContainerStatuses(); if (containerStatuses != null) { for (ContainerStatus containerStatus : containerStatuses) { String containerID = containerStatus.getContainerID(); if (StringUtils.isNotBlank(containerID)) { String prefix = "://"; int idx = containerID.indexOf(prefix); if (idx > 0) { return containerID.substring(idx + prefix.length()); } return containerID; } } } } return null; }
@Override public boolean test(ContainerStatus containerStatus) { // if we are being killed repeatedly due to OOM or using too much CPU, or abnormal termination. return containerStatus.getLastState() != null && containerStatus.getLastState().getTerminated() != null && (containerStatus.getLastState().getTerminated().getExitCode() == 137 || containerStatus.getLastState().getTerminated().getExitCode() == 143 || containerStatus.getLastState().getTerminated().getExitCode() == 1); } });
public ContainerStatusFluentImpl(ContainerStatus instance){ this.withContainerID(instance.getContainerID()); this.withImage(instance.getImage()); this.withImageID(instance.getImageID()); this.withLastState(instance.getLastState()); this.withName(instance.getName()); this.withReady(instance.getReady()); this.withRestartCount(instance.getRestartCount()); this.withState(instance.getState()); }
for (ContainerStatus info : containerStatuses) { if (info != null) { if (info.getState().getWaiting() != null) { new Object[]{podId, info.getState().getWaiting(), info.getName()}); logger.printf("Container is waiting %1$s [%3$s]: %2$s%n", podId, info.getState().getWaiting(), info.getName()); if (info.getState().getTerminated() != null) { terminatedContainers.add(info); } else if (!info.getReady()) { allContainersAreReady = false; .toMap(ContainerStatus::getName, (info) -> info.getState().getTerminated().getExitCode()));
public boolean test(ContainerStatus containerStatus) { return containerStatus.getLastState() != null && containerStatus.getState() != null && containerStatus.getLastState().getTerminated() != null && containerStatus.getLastState().getTerminated().getReason() != null && containerStatus.getLastState().getTerminated().getReason().contains("Error") && containerStatus.getState().getTerminated() != null && containerStatus.getState().getTerminated().getReason() != null && containerStatus.getState().getTerminated().getReason().contains("Error"); } }, new ContainerStatusCondition("last state exit code == exit code") {