254 lines
7.8 KiB
Python
254 lines
7.8 KiB
Python
import logging
|
|
import pytest
|
|
import re
|
|
import shutil
|
|
import xml.etree.ElementTree as ET
|
|
|
|
from os import environ, chdir
|
|
from os.path import dirname, exists, realpath
|
|
|
|
import tracie
|
|
|
|
|
|
RESULTS_YAML = "results/results.yml"
|
|
JUNIT_XML = "results/junit.xml"
|
|
TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
|
|
TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
|
|
TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
|
|
TRACE_PNG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace-0.png"
|
|
TRACIE_DIR = dirname(realpath(__file__)) + "/.."
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger()
|
|
|
|
|
|
def write_to(content, filename):
|
|
with open(filename, 'w') as f:
|
|
f.write(content)
|
|
|
|
|
|
def read_from(filename):
|
|
with open(filename) as f:
|
|
content = f.read()
|
|
return content
|
|
|
|
|
|
def run_tracie():
|
|
'''
|
|
Run tests for the .testtrace types, using the "gl-test-device" and
|
|
"vk-test-device" device names.
|
|
'''
|
|
result = tracie.main(["--device-name", "gl-test-device",
|
|
"--file", "./tests/traces.yml"])
|
|
if not result:
|
|
return False
|
|
result = tracie.main(["--device-name", "vk-test-device",
|
|
"--file", "./tests/traces.yml"])
|
|
return result
|
|
|
|
|
|
def prepare_for_run(tmp_path):
|
|
'''
|
|
Copy all the tracie scripts to the test dir for the unit tests.
|
|
This avoids polluting the normal working dir with test result artifacts.
|
|
'''
|
|
test_dir = str(tmp_path) + "/run"
|
|
shutil.copytree(TRACIE_DIR, test_dir)
|
|
# Change the working dir to the test_dir
|
|
chdir(test_dir)
|
|
# Set the traces-db
|
|
shutil.move("./tests/test-data", "./traces-db")
|
|
# Disable trace storing
|
|
environ["TRACIE_STORE_IMAGES"] = "0"
|
|
environ["TRACIE_UPLOAD_TO_MINIO"] = "0"
|
|
environ["CI_PROJECT_PATH"] = "test-project"
|
|
environ["CI_PIPELINE_ID"] = "667"
|
|
environ["CI_JOB_ID"] = "42"
|
|
|
|
def cleanup(tmp_path):
|
|
'''
|
|
Performs the clean up of the test dir.
|
|
'''
|
|
if exists(tmp_path):
|
|
shutil.rmtree(tmp_path)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def run_test(tmp_path):
|
|
'''
|
|
Wraps the execution of each test as follows:
|
|
|
|
prepare_for_run()
|
|
test()
|
|
cleanup()
|
|
'''
|
|
logger.debug("Working dir: %s", tmp_path)
|
|
prepare_for_run(tmp_path)
|
|
yield
|
|
cleanup(tmp_path)
|
|
|
|
|
|
def check_results_yaml_content(filename, expectations):
|
|
'''
|
|
Checks the content of the filename with the list of expectations
|
|
passed as parameter.
|
|
|
|
Arguments:
|
|
filename (str): The path of the file to check
|
|
expectations (list): A list with the content to find in the file
|
|
|
|
Returns:
|
|
bool: The return value. True if the content of the filename satisfies
|
|
the expectations, False otherwise.
|
|
'''
|
|
content = read_from(filename)
|
|
for e in expectations:
|
|
ocurrencies = re.findall(e, content)
|
|
if not len(ocurrencies):
|
|
logger.error("Expectation not found in %s: %s", filename, e)
|
|
return False
|
|
return True
|
|
|
|
|
|
def test_tracie_succeeds_if_all_images_match():
|
|
assert run_tracie()
|
|
expectations = [
|
|
"actual: 5efda83854befe0155ff8517a58d5b51",
|
|
"expected: 5efda83854befe0155ff8517a58d5b51",
|
|
]
|
|
assert check_results_yaml_content(RESULTS_YAML, expectations)
|
|
|
|
|
|
def test_tracie_fails_on_image_mismatch():
|
|
filename = "./tests/traces.yml"
|
|
content = read_from(filename)
|
|
content = content.replace("5efda83854befe0155ff8517a58d5b51",
|
|
"8e0a801367e1714463475a824dab363b")
|
|
write_to(content, filename)
|
|
assert not run_tracie()
|
|
expectations = [
|
|
"actual: 5efda83854befe0155ff8517a58d5b51",
|
|
"expected: 8e0a801367e1714463475a824dab363b",
|
|
"trace2/test/vk-test-device/olive.testtrace-0.png"
|
|
]
|
|
assert check_results_yaml_content(RESULTS_YAML, expectations)
|
|
|
|
|
|
def test_tracie_traces_with_and_without_checksum():
|
|
filename = "./tests/traces.yml"
|
|
content = read_from(filename)
|
|
content += ''' - path: trace1/red.testtrace
|
|
expectations:
|
|
- device: bla
|
|
checksum: 000000000000000'''
|
|
write_to(content, filename)
|
|
|
|
# red.testtrace should be skipped, since it doesn't
|
|
# have any checksums for our device
|
|
filename = "./traces-db/trace1/red.testtrace"
|
|
content = "ff0000ff"
|
|
write_to(content, filename)
|
|
assert run_tracie()
|
|
|
|
|
|
def test_tracie_only_traces_without_checksum():
|
|
filename = "./tests/traces.yml"
|
|
content = '''traces:
|
|
- path: trace1/red.testtrace
|
|
expectations:
|
|
- device: bla
|
|
checksum: 000000000000000'''
|
|
write_to(content, filename)
|
|
|
|
# red.testtrace should be skipped, since it doesn't
|
|
# have any checksums for our device
|
|
filename = "./traces-db/trace1/red.testtrace"
|
|
content = "ff0000ff"
|
|
write_to(content, filename)
|
|
assert run_tracie()
|
|
|
|
|
|
def test_tracie_with_no_traces():
|
|
filename = "./tests/traces.yml"
|
|
content = 'traces:'
|
|
write_to(content, filename)
|
|
assert run_tracie()
|
|
expectations = [
|
|
"{}",
|
|
]
|
|
assert check_results_yaml_content(RESULTS_YAML, expectations)
|
|
|
|
|
|
def test_tracie_fails_on_dump_image_error():
|
|
# "invalid" should fail to parse as rgba and
|
|
# cause an error
|
|
filename = "./traces-db/trace1/magenta.testtrace"
|
|
write_to("invalid\n", filename)
|
|
run_tracie()
|
|
expectations = [
|
|
"actual: error",
|
|
"expected: 8e0a801367e1714463475a824dab363b",
|
|
"trace1/magenta.testtrace",
|
|
]
|
|
assert check_results_yaml_content(RESULTS_YAML, expectations)
|
|
|
|
|
|
def test_tracie_stores_only_logs_on_checksum_match():
|
|
assert run_tracie()
|
|
assert exists(TRACE_LOG_TEST1)
|
|
assert exists(TRACE_LOG_TEST2)
|
|
assert not exists(TRACE_PNG_TEST1)
|
|
assert not exists(TRACE_PNG_TEST2)
|
|
|
|
|
|
def test_tracie_stores_images_on_checksum_mismatch():
|
|
filename = "./tests/traces.yml"
|
|
content = read_from(filename)
|
|
content = content.replace("5efda83854befe0155ff8517a58d5b51",
|
|
"8e0a801367e1714463475a824dab363b")
|
|
write_to(content, filename)
|
|
assert not run_tracie()
|
|
assert not exists(TRACE_PNG_TEST1)
|
|
assert exists(TRACE_PNG_TEST2)
|
|
|
|
|
|
def test_tracie_stores_images_on_request():
|
|
environ["TRACIE_STORE_IMAGES"] = "1"
|
|
assert run_tracie()
|
|
assert exists(TRACE_PNG_TEST1)
|
|
assert exists(TRACE_PNG_TEST2)
|
|
|
|
def test_tracie_writes_junit_xml():
|
|
assert run_tracie()
|
|
junit_xml = ET.parse(JUNIT_XML)
|
|
assert junit_xml.getroot().tag == 'testsuites'
|
|
testsuites = junit_xml.findall("./testsuite")
|
|
testcases_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase")
|
|
testcases_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase")
|
|
|
|
assert len(testsuites) == 2
|
|
assert len(testcases_gl) == 1
|
|
assert len(testcases_vk) == 1
|
|
assert testcases_gl[0].get("name") == "trace1/magenta.testtrace"
|
|
assert testcases_gl[0].get("classname") == "traces.yml:gl-test-device"
|
|
assert testcases_vk[0].get("name") == "trace2/olive.testtrace"
|
|
assert testcases_vk[0].get("classname") == "traces.yml:vk-test-device"
|
|
|
|
def test_tracie_writes_dashboard_url_in_junit_xml_failure_tag():
|
|
filename = "./tests/traces.yml"
|
|
content = read_from(filename)
|
|
content = content.replace("5efda83854befe0155ff8517a58d5b51",
|
|
"8e0a801367e1714463475a824dab363b")
|
|
write_to(content, filename)
|
|
|
|
assert not run_tracie()
|
|
|
|
junit_xml = ET.parse(JUNIT_XML)
|
|
failures_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase/failure")
|
|
failures_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase/failure")
|
|
|
|
assert len(failures_gl) == 0
|
|
assert len(failures_vk) == 1
|
|
dashboard_url = "https://tracie.freedesktop.org/dashboard/imagediff/test-project/42/trace2/olive.testtrace"
|
|
assert dashboard_url in failures_vk[0].text
|