ci/lava: Parse all test cases from 0_mesa suite

LAVA can filter which test suite to show the results from, let's list
all testcases possible in the mesa test suite, to be able to divide more
complex jobs into test_cases.
Another advantage is that the test case can vary its name.

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15938>
This commit is contained in:
Guilherme Gallo 2022-03-23 22:10:51 -03:00 committed by Marge Bot
parent e3f71aaa37
commit 18d80f25ee
2 changed files with 94 additions and 35 deletions

View File

@ -267,7 +267,7 @@ class LAVAJob():
fatal_err(f"Could not get LAVA job logs. Reason: {mesa_exception}")
def get_job_results(proxy, job_id, test_suite, test_case):
def get_job_results(proxy, job_id, test_suite):
# Look for infrastructure errors and retry if we see them.
results_yaml = _call_proxy(proxy.results.get_testjob_results_yaml, job_id)
results = yaml.load(results_yaml, Loader=loader(False))
@ -275,19 +275,33 @@ def get_job_results(proxy, job_id, test_suite, test_case):
metadata = res["metadata"]
if "result" not in metadata or metadata["result"] != "fail":
continue
if 'error_type' in metadata and metadata['error_type'] == "Infrastructure":
raise MesaCIException("LAVA job {} failed with Infrastructure Error. Retry.".format(job_id))
if 'case' in metadata and metadata['case'] == "validate":
raise MesaCIException("LAVA job {} failed validation (possible download error). Retry.".format(job_id))
if "error_type" in metadata and metadata["error_type"] == "Infrastructure":
raise MesaCIException(
f"LAVA job {job_id} failed with Infrastructure Error. Retry."
)
if "case" in metadata and metadata["case"] == "validate":
raise MesaCIException(
f"LAVA job {job_id} failed validation (possible download error). Retry."
)
results_yaml = _call_proxy(proxy.results.get_testcase_results_yaml, job_id, test_suite, test_case)
results = yaml.load(results_yaml, Loader=loader(False))
results_yaml = _call_proxy(
proxy.results.get_testsuite_results_yaml, job_id, test_suite
)
results: list = yaml.load(results_yaml, Loader=loader(False))
if not results:
raise MesaCIException("LAVA: no result for test_suite '{}', test_case '{}'".format(test_suite, test_case))
raise MesaCIException(
f"LAVA: no result for test_suite '{test_suite}'"
)
print_log("LAVA: result for test_suite '{}', test_case '{}': {}".format(test_suite, test_case, results[0]['result']))
if results[0]['result'] != 'pass':
return False
for metadata in results:
test_case = metadata["name"]
result = metadata["metadata"]["result"]
print_log(
f"LAVA: result for test_suite '{test_suite}', "
f"test_case '{test_case}': {result}"
)
if result != "pass":
return False
return True
@ -353,7 +367,7 @@ def follow_job_execution(job):
print(line)
show_job_data(job)
return get_job_results(job.proxy, job.job_id, "0_mesa", "mesa")
return get_job_results(job.proxy, job.job_id, "0_mesa")
def retriable_follow_job(proxy, job_definition):

View File

@ -26,19 +26,20 @@ import xmlrpc.client
from contextlib import nullcontext as does_not_raise
from datetime import datetime
from itertools import cycle, repeat
from typing import Iterable, Union, Generator, Tuple
from typing import Generator, Iterable, Tuple, Union
from unittest.mock import MagicMock, patch
import pytest
import yaml
from freezegun import freeze_time
from lava.lava_job_submitter import (
NUMBER_OF_RETRIES_TIMEOUT_DETECTION,
DEVICE_HANGING_TIMEOUT_SEC,
NUMBER_OF_RETRIES_TIMEOUT_DETECTION,
LAVAJob,
follow_job_execution,
get_job_results,
hide_sensitive_data,
retriable_follow_job,
LAVAJob
)
NUMBER_OF_MAX_ATTEMPTS = NUMBER_OF_RETRIES_TIMEOUT_DETECTION + 1
@ -51,35 +52,37 @@ def jobs_logs_response(finished=False, msg=None, lvl="target") -> Tuple[bool, st
return finished, yaml.safe_dump(logs)
def result_get_testjob_results_response() -> str:
result = {"result": "test"}
results = [{"metadata": result}]
return yaml.safe_dump(results)
RESULT_GET_TESTJOB_RESULTS = [{"metadata": {"result": "test"}}]
def result_get_testcase_results_response() -> str:
result = {"result": "pass"}
test_cases = [result]
return yaml.safe_dump(test_cases)
def generate_testsuite_result(name="test-mesa-ci", result="pass", metadata_extra = None, extra = None):
if metadata_extra is None:
metadata_extra = {}
if extra is None:
extra = {}
return {"metadata": {"result": result, **metadata_extra}, "name": name}
@pytest.fixture
def mock_proxy():
def create_proxy_mock(**kwargs):
def create_proxy_mock(
job_results=RESULT_GET_TESTJOB_RESULTS,
testsuite_results=[generate_testsuite_result()],
**kwargs
):
proxy_mock = MagicMock()
proxy_submit_mock = proxy_mock.scheduler.jobs.submit
proxy_submit_mock.return_value = "1234"
proxy_results_mock = proxy_mock.results.get_testjob_results_yaml
proxy_results_mock.return_value = result_get_testjob_results_response()
proxy_results_mock.return_value = yaml.safe_dump(job_results)
proxy_test_cases_mock = proxy_mock.results.get_testcase_results_yaml
proxy_test_cases_mock.return_value = result_get_testcase_results_response()
proxy_test_suites_mock = proxy_mock.results.get_testsuite_results_yaml
proxy_test_suites_mock.return_value = yaml.safe_dump(testsuite_results)
proxy_logs_mock = proxy_mock.scheduler.jobs.logs
proxy_logs_mock.return_value = jobs_logs_response()
for key, value in kwargs.items():
setattr(proxy_logs_mock, key, value)
@ -156,56 +159,93 @@ NETWORK_EXCEPTION = xmlrpc.client.ProtocolError("", 0, "test", {})
XMLRPC_FAULT = xmlrpc.client.Fault(0, "test")
PROXY_SCENARIOS = {
"finish case": (generate_n_logs(1), does_not_raise(), True),
"finish case": (generate_n_logs(1), does_not_raise(), True, {}),
"works at last retry": (
generate_n_logs(n=NUMBER_OF_MAX_ATTEMPTS, tick_fn=[ DEVICE_HANGING_TIMEOUT_SEC + 1 ] * NUMBER_OF_RETRIES_TIMEOUT_DETECTION + [1]),
does_not_raise(),
True,
{},
),
"timed out more times than retry attempts": (
generate_n_logs(n=4, tick_fn=DEVICE_HANGING_TIMEOUT_SEC + 1),
pytest.raises(SystemExit),
False,
{},
),
"long log case, no silence": (
generate_n_logs(n=1000, tick_fn=0),
does_not_raise(),
True,
{},
),
"no retries, testsuite succeed": (
generate_n_logs(n=1, tick_fn=0),
does_not_raise(),
True,
{
"testsuite_results": [
generate_testsuite_result(result="pass")
]
},
),
"no retries, but testsuite fails": (
generate_n_logs(n=1, tick_fn=0),
does_not_raise(),
False,
{
"testsuite_results": [
generate_testsuite_result(result="fail")
]
},
),
"no retries, one testsuite fails": (
generate_n_logs(n=1, tick_fn=0),
does_not_raise(),
False,
{
"testsuite_results": [
generate_testsuite_result(result="fail"),
generate_testsuite_result(result="pass")
]
},
),
"very long silence": (
generate_n_logs(n=NUMBER_OF_MAX_ATTEMPTS + 1, tick_fn=100000),
pytest.raises(SystemExit),
False,
{},
),
# If a protocol error happens, _call_proxy will retry without affecting timeouts
"unstable connection, ProtocolError followed by final message": (
(NETWORK_EXCEPTION, jobs_logs_response(finished=True)),
does_not_raise(),
True,
{},
),
# After an arbitrary number of retries, _call_proxy should call sys.exit
"unreachable case, subsequent ProtocolErrors": (
repeat(NETWORK_EXCEPTION),
pytest.raises(SystemExit),
False,
{},
),
"XMLRPC Fault": ([XMLRPC_FAULT], pytest.raises(SystemExit, match="1"), False),
"XMLRPC Fault": ([XMLRPC_FAULT], pytest.raises(SystemExit, match="1"), False, {}),
}
@patch("time.sleep", return_value=None) # mock sleep to make test faster
@pytest.mark.parametrize(
"side_effect, expectation, has_finished",
"side_effect, expectation, job_result, proxy_args",
PROXY_SCENARIOS.values(),
ids=PROXY_SCENARIOS.keys(),
)
def test_retriable_follow_job(
mock_sleep, side_effect, expectation, has_finished, mock_proxy
mock_sleep, side_effect, expectation, job_result, proxy_args, mock_proxy
):
with expectation:
proxy = mock_proxy(side_effect=side_effect)
proxy = mock_proxy(side_effect=side_effect, **proxy_args)
result = retriable_follow_job(proxy, "")
assert has_finished == result
assert job_result == result
WAIT_FOR_JOB_SCENARIOS = {
@ -271,3 +311,8 @@ def test_hide_sensitive_data(input, expectation, tag):
result = yaml.safe_load(yaml_result)
assert result == expectation
def test_get_job_results(mock_proxy):
proxy = mock_proxy()
get_job_results(proxy, 1, "0_mesa")