From 70f1291d8e4f58611e21489d73ac3d96ab3a9b14 Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Wed, 6 Jul 2022 23:51:26 -0300 Subject: [PATCH] ci/lava: Add canceled job status We should be explicit that we are cancelling jobs once the script finds some log messages that are linked with known issues. That means the script preemptively retried the job without giving chances to recover. Adds magenta color to cancelled jobs. Signed-off-by: Guilherme Gallo Part-of: --- .gitlab-ci/lava/lava_job_submitter.py | 17 +++++++++++++++-- .gitlab-ci/lava/utils/console_format.py | 1 + .gitlab-ci/lava/utils/lava_log_hints.py | 3 +++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index 1bbd77d8e85..5d9c22749b1 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -27,6 +27,7 @@ import lavacli import yaml from lava.exceptions import ( MesaCIException, + MesaCIKnownIssueException, MesaCIParseException, MesaCIRetryError, MesaCITimeoutError, @@ -221,6 +222,7 @@ class LAVAJob: "pass": CONSOLE_LOG["FG_GREEN"], "hung": CONSOLE_LOG["FG_YELLOW"], "fail": CONSOLE_LOG["FG_RED"], + "canceled": CONSOLE_LOG["FG_MAGENTA"], } def __init__(self, proxy, definition): @@ -447,6 +449,9 @@ def retriable_follow_job(proxy, job_definition) -> LAVAJob: try: follow_job_execution(job) return job + except MesaCIKnownIssueException as found_issue: + print_log(found_issue) + job.status = "canceled" except MesaCIException as mesa_exception: print_log(mesa_exception) job.cancel() @@ -455,11 +460,19 @@ def retriable_follow_job(proxy, job_definition) -> LAVAJob: job.cancel() raise e finally: - print_log(f"Finished executing LAVA job in the attempt #{attempt_no}") + print_log( + f"{CONSOLE_LOG['BOLD']}" + f"Finished executing LAVA job in the attempt #{attempt_no}" + f"{CONSOLE_LOG['RESET']}" + ) print_job_final_status(job) raise MesaCIRetryError( - "Job failed after it exceeded the number of " f"{retry_count} retries.", + f"{CONSOLE_LOG['BOLD']}" + f"{CONSOLE_LOG['FG_RED']}" + "Job failed after it exceeded the number of " + f"{retry_count} retries." + f"{CONSOLE_LOG['RESET']}", retry_count=retry_count, ) diff --git a/.gitlab-ci/lava/utils/console_format.py b/.gitlab-ci/lava/utils/console_format.py index a665b2c23aa..3ad7600591b 100644 --- a/.gitlab-ci/lava/utils/console_format.py +++ b/.gitlab-ci/lava/utils/console_format.py @@ -2,6 +2,7 @@ CONSOLE_LOG = { "FG_GREEN": "\x1b[1;32;5;197m", "FG_RED": "\x1b[1;38;5;197m", "FG_YELLOW": "\x1b[1;33;5;197m", + "FG_MAGENTA": "\x1b[1;35;5;197m", "RESET": "\x1b[0m", "UNDERLINED": "\x1b[3m", "BOLD": "\x1b[1m", diff --git a/.gitlab-ci/lava/utils/lava_log_hints.py b/.gitlab-ci/lava/utils/lava_log_hints.py index 12317bb1127..b147a8747ea 100644 --- a/.gitlab-ci/lava/utils/lava_log_hints.py +++ b/.gitlab-ci/lava/utils/lava_log_hints.py @@ -8,6 +8,7 @@ if TYPE_CHECKING: from lava.utils import LogFollower from lava.exceptions import MesaCIKnownIssueException +from lava.utils.console_format import CONSOLE_LOG from lava.utils.log_section import LogSectionType @@ -34,7 +35,9 @@ class LAVALogHints: line["msg"], ): raise MesaCIKnownIssueException( + f"{CONSOLE_LOG['FG_MAGENTA']}" "Probable network issue failure encountered, retrying the job" + f"{CONSOLE_LOG['RESET']}" ) self.has_r8152_issue_history = False