ci/lava: Stop printing after the result line
There are some leftovers in the jobs logs after the result log line. Only print until the init-stage2.sh output, to raise the chance to check for the test script results at the first glance in the Gitlab logs. Extra changes: - Add `hung` status for jobs considered hanging in the Gitlab - print them after the retry loop Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16323>
This commit is contained in:
parent
f09aab08e9
commit
24f368d652
|
@ -294,28 +294,23 @@ class LAVAJob:
|
||||||
f"Could not get LAVA job logs. Reason: {mesa_ci_err}"
|
f"Could not get LAVA job logs. Reason: {mesa_ci_err}"
|
||||||
) from mesa_ci_err
|
) from mesa_ci_err
|
||||||
|
|
||||||
def parse_job_result_from_log(self, lava_lines: list[dict[str, str]]) -> None:
|
def parse_job_result_from_log(
|
||||||
|
self, lava_lines: list[dict[str, str]]
|
||||||
|
) -> list[dict[str, str]]:
|
||||||
"""Use the console log to catch if the job has completed successfully or
|
"""Use the console log to catch if the job has completed successfully or
|
||||||
not.
|
not. Returns the list of log lines until the result line."""
|
||||||
Returns true only the job finished by looking into the log result
|
|
||||||
parsing.
|
last_line = None # Print all lines. lines[:None] == lines[:]
|
||||||
"""
|
|
||||||
log_lines = [l["msg"] for l in lava_lines if l["lvl"] == "target"]
|
for idx, line in enumerate(lava_lines):
|
||||||
for line in log_lines:
|
|
||||||
if result := re.search(r"hwci: mesa: (pass|fail)", line):
|
if result := re.search(r"hwci: mesa: (pass|fail)", line):
|
||||||
self.is_finished = True
|
self.is_finished = True
|
||||||
self.status = result.group(1)
|
self.status = result.group(1)
|
||||||
color = LAVAJob.COLOR_STATUS_MAP.get(
|
|
||||||
self.status, CONSOLE_LOG["COLOR_RED"]
|
|
||||||
)
|
|
||||||
print_log(
|
|
||||||
f"{color}"
|
|
||||||
f"LAVA Job finished with result: {self.status}"
|
|
||||||
f"{CONSOLE_LOG['RESET']}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
last_line = idx + 1
|
||||||
# We reached the log end here. hwci script has finished.
|
# We reached the log end here. hwci script has finished.
|
||||||
break
|
break
|
||||||
|
return lava_lines[:last_line]
|
||||||
|
|
||||||
|
|
||||||
def find_exception_from_metadata(metadata, job_id):
|
def find_exception_from_metadata(metadata, job_id):
|
||||||
|
@ -403,11 +398,11 @@ def fetch_logs(job, max_idle_time, log_follower) -> None:
|
||||||
job.heartbeat()
|
job.heartbeat()
|
||||||
parsed_lines = log_follower.flush()
|
parsed_lines = log_follower.flush()
|
||||||
|
|
||||||
|
parsed_lines = job.parse_job_result_from_log(parsed_lines)
|
||||||
|
|
||||||
for line in parsed_lines:
|
for line in parsed_lines:
|
||||||
print_log(line)
|
print_log(line)
|
||||||
|
|
||||||
job.parse_job_result_from_log(new_log_lines)
|
|
||||||
|
|
||||||
|
|
||||||
def follow_job_execution(job):
|
def follow_job_execution(job):
|
||||||
try:
|
try:
|
||||||
|
@ -447,6 +442,18 @@ def follow_job_execution(job):
|
||||||
find_lava_error(job)
|
find_lava_error(job)
|
||||||
|
|
||||||
|
|
||||||
|
def print_job_final_status(job):
|
||||||
|
if job.status == "running":
|
||||||
|
job.status = "hung"
|
||||||
|
|
||||||
|
color = LAVAJob.COLOR_STATUS_MAP.get(job.status, CONSOLE_LOG["COLOR_RED"])
|
||||||
|
print_log(
|
||||||
|
f"{color}"
|
||||||
|
f"LAVA Job finished with status: {job.status}"
|
||||||
|
f"{CONSOLE_LOG['RESET']}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def retriable_follow_job(proxy, job_definition) -> LAVAJob:
|
def retriable_follow_job(proxy, job_definition) -> LAVAJob:
|
||||||
retry_count = NUMBER_OF_RETRIES_TIMEOUT_DETECTION
|
retry_count = NUMBER_OF_RETRIES_TIMEOUT_DETECTION
|
||||||
|
|
||||||
|
@ -464,6 +471,7 @@ def retriable_follow_job(proxy, job_definition) -> LAVAJob:
|
||||||
raise e
|
raise e
|
||||||
finally:
|
finally:
|
||||||
print_log(f"Finished executing LAVA job in the attempt #{attempt_no}")
|
print_log(f"Finished executing LAVA job in the attempt #{attempt_no}")
|
||||||
|
print_job_final_status(job)
|
||||||
|
|
||||||
raise MesaCIRetryError(
|
raise MesaCIRetryError(
|
||||||
"Job failed after it exceeded the number of " f"{retry_count} retries.",
|
"Job failed after it exceeded the number of " f"{retry_count} retries.",
|
||||||
|
|
|
@ -259,34 +259,22 @@ def test_log_corruption(mock_sleep, data_sequence, expected_exception, mock_prox
|
||||||
LAVA_RESULT_LOG_SCENARIOS = {
|
LAVA_RESULT_LOG_SCENARIOS = {
|
||||||
# the submitter should accept xtrace logs
|
# the submitter should accept xtrace logs
|
||||||
"Bash xtrace echo with kmsg interleaving": (
|
"Bash xtrace echo with kmsg interleaving": (
|
||||||
create_lava_yaml_msg(
|
"echo hwci: mesa: pass[ 737.673352] <LAVA_SIGNAL_ENDTC mesa-ci>",
|
||||||
msg="echo hwci: mesa: pass[ 737.673352] <LAVA_SIGNAL_ENDTC mesa-ci>",
|
|
||||||
lvl="target",
|
|
||||||
),
|
|
||||||
"pass",
|
"pass",
|
||||||
),
|
),
|
||||||
# the submitter should accept xtrace logs
|
# the submitter should accept xtrace logs
|
||||||
"kmsg result print": (
|
"kmsg result print": (
|
||||||
create_lava_yaml_msg(
|
"[ 737.673352] hwci: mesa: pass",
|
||||||
msg="[ 737.673352] hwci: mesa: pass",
|
|
||||||
lvl="target",
|
|
||||||
),
|
|
||||||
"pass",
|
"pass",
|
||||||
),
|
),
|
||||||
# if the job result echo has a very bad luck, it still can be interleaved
|
# if the job result echo has a very bad luck, it still can be interleaved
|
||||||
# with kmsg
|
# with kmsg
|
||||||
"echo output with kmsg interleaving": (
|
"echo output with kmsg interleaving": (
|
||||||
create_lava_yaml_msg(
|
"hwci: mesa: pass[ 737.673352] <LAVA_SIGNAL_ENDTC mesa-ci>",
|
||||||
msg="hwci: mesa: pass[ 737.673352] <LAVA_SIGNAL_ENDTC mesa-ci>",
|
|
||||||
lvl="target",
|
|
||||||
),
|
|
||||||
"pass",
|
"pass",
|
||||||
),
|
),
|
||||||
"fail case": (
|
"fail case": (
|
||||||
create_lava_yaml_msg(
|
"hwci: mesa: fail",
|
||||||
msg="hwci: mesa: fail",
|
|
||||||
lvl="target",
|
|
||||||
),
|
|
||||||
"fail",
|
"fail",
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
@ -297,7 +285,7 @@ LAVA_RESULT_LOG_SCENARIOS = {
|
||||||
LAVA_RESULT_LOG_SCENARIOS.values(),
|
LAVA_RESULT_LOG_SCENARIOS.values(),
|
||||||
ids=LAVA_RESULT_LOG_SCENARIOS.keys(),
|
ids=LAVA_RESULT_LOG_SCENARIOS.keys(),
|
||||||
)
|
)
|
||||||
def test_filter_debug_messages(message, expectation, mock_proxy):
|
def test_parse_job_result_from_log(message, expectation, mock_proxy):
|
||||||
job = LAVAJob(mock_proxy(), "")
|
job = LAVAJob(mock_proxy(), "")
|
||||||
job.parse_job_result_from_log([message])
|
job.parse_job_result_from_log([message])
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue