mirror of https://gitlab.freedesktop.org/mesa/mesa
ci: Update the skqp testing docs and retire the old runner script.
Reviewed-by: Guilherme Gallo <guilherme.gallo@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20070>
This commit is contained in:
Emma Anholt
Marge Bot
parent
0cff5d51ac
commit
58e6d8eee2
|
|
@ -1,371 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (C) 2022 Collabora Limited
|
||||
# Author: Guilherme Gallo <guilherme.gallo@collabora.com>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice (including the next
|
||||
# paragraph) shall be included in all copies or substantial portions of the
|
||||
# Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Args:
|
||||
# $1: section id
|
||||
# $2: section header
|
||||
gitlab_section_start() {
|
||||
echo -e "\e[0Ksection_start:$(date +%s):$1[collapsed=${GL_COLLAPSED:-false}]\r\e[0K\e[32;1m$2\e[0m"
|
||||
}
|
||||
|
||||
# Args:
|
||||
# $1: section id
|
||||
gitlab_section_end() {
|
||||
echo -e "\e[0Ksection_end:$(date +%s):$1\r\e[0K"
|
||||
}
|
||||
|
||||
|
||||
# sponge allows piping to files that are being used as input.
|
||||
# E.g.: sort file.txt | sponge file.txt
|
||||
# In order to avoid installing moreutils just to have the sponge binary, we can
|
||||
# use a bash function for it
|
||||
# Source https://unix.stackexchange.com/a/561346/310927
|
||||
sponge () (
|
||||
set +x
|
||||
append=false
|
||||
|
||||
while getopts 'a' opt; do
|
||||
case $opt in
|
||||
a) append=true ;;
|
||||
*) echo error; exit 1
|
||||
esac
|
||||
done
|
||||
shift "$(( OPTIND - 1 ))"
|
||||
|
||||
outfile=$1
|
||||
|
||||
tmpfile=$(mktemp "$(dirname "$outfile")/tmp-sponge.XXXXXXXX") &&
|
||||
cat >"$tmpfile" &&
|
||||
if "$append"; then
|
||||
cat "$tmpfile" >>"$outfile"
|
||||
else
|
||||
if [ -f "$outfile" ]; then
|
||||
chmod --reference="$outfile" "$tmpfile"
|
||||
fi
|
||||
if [ -f "$outfile" ]; then
|
||||
mv "$tmpfile" "$outfile"
|
||||
elif [ -n "$outfile" ] && [ ! -e "$outfile" ]; then
|
||||
cat "$tmpfile" >"$outfile"
|
||||
else
|
||||
cat "$tmpfile"
|
||||
fi
|
||||
fi &&
|
||||
rm -f "$tmpfile"
|
||||
)
|
||||
|
||||
remove_comments_from_files() (
|
||||
INPUT_FILES="$*"
|
||||
for INPUT_FILE in ${INPUT_FILES}
|
||||
do
|
||||
[ -f "${INPUT_FILE}" ] || continue
|
||||
sed -i '/#/d' "${INPUT_FILE}"
|
||||
sed -i '/^\s*$/d' "${INPUT_FILE}"
|
||||
done
|
||||
)
|
||||
|
||||
subtract_test_lists() (
|
||||
MINUEND=$1
|
||||
sort "${MINUEND}" | sponge "${MINUEND}"
|
||||
shift
|
||||
for SUBTRAHEND in "$@"
|
||||
do
|
||||
sort "${SUBTRAHEND}" | sponge "${SUBTRAHEND}"
|
||||
join -v 1 "${MINUEND}" "${SUBTRAHEND}" |
|
||||
sponge "${MINUEND}"
|
||||
done
|
||||
)
|
||||
|
||||
merge_rendertests_files() {
|
||||
BASE_FILE=$1
|
||||
shift
|
||||
FILES="$*"
|
||||
# shellcheck disable=SC2086
|
||||
cat $FILES "$BASE_FILE" |
|
||||
sort --unique --stable --field-separator=, --key=1,1 |
|
||||
sponge "$BASE_FILE"
|
||||
}
|
||||
|
||||
assure_files() (
|
||||
for CASELIST_FILE in $*
|
||||
do
|
||||
>&2 echo "Looking for ${CASELIST_FILE}..."
|
||||
[ -f ${CASELIST_FILE} ] || (
|
||||
>&2 echo "Not found. Creating empty."
|
||||
touch ${CASELIST_FILE}
|
||||
)
|
||||
done
|
||||
)
|
||||
|
||||
# Generate rendertests from scratch, customizing with fails/flakes/crashes files
|
||||
generate_rendertests() (
|
||||
set -e
|
||||
GENERATED_FILE=$(mktemp)
|
||||
TESTS_FILE_PREFIX="${SKQP_FILE_PREFIX}-${SKQP_BACKEND}_rendertests"
|
||||
FLAKES_FILE="${TESTS_FILE_PREFIX}-flakes.txt"
|
||||
FAILS_FILE="${TESTS_FILE_PREFIX}-fails.txt"
|
||||
CRASHES_FILE="${TESTS_FILE_PREFIX}-crashes.txt"
|
||||
RENDER_TESTS_FILE="${TESTS_FILE_PREFIX}.txt"
|
||||
|
||||
# Default to an empty known flakes file if it doesn't exist.
|
||||
assure_files ${FLAKES_FILE} ${FAILS_FILE} ${CRASHES_FILE}
|
||||
|
||||
# skqp does not support comments in rendertests.txt file
|
||||
remove_comments_from_files "${FLAKES_FILE}" "${FAILS_FILE}" "${CRASHES_FILE}"
|
||||
|
||||
# create an exhaustive rendertest list
|
||||
"${SKQP_BIN_DIR}"/list_gms | sort > "$GENERATED_FILE"
|
||||
|
||||
# Remove undesirable tests from the list
|
||||
subtract_test_lists "${GENERATED_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}"
|
||||
|
||||
# Add ",0" to each test to set the expected diff sum to zero
|
||||
sed -i 's/$/,0/g' "$GENERATED_FILE"
|
||||
|
||||
merge_rendertests_files "$GENERATED_FILE" "${FAILS_FILE}"
|
||||
|
||||
mv "${GENERATED_FILE}" "${RENDER_TESTS_FILE}"
|
||||
|
||||
echo "${RENDER_TESTS_FILE}"
|
||||
)
|
||||
|
||||
generate_unittests() (
|
||||
set -e
|
||||
GENERATED_FILE=$(mktemp)
|
||||
TESTS_FILE_PREFIX="${SKQP_FILE_PREFIX}_unittests"
|
||||
FLAKES_FILE="${TESTS_FILE_PREFIX}-flakes.txt"
|
||||
FAILS_FILE="${TESTS_FILE_PREFIX}-fails.txt"
|
||||
CRASHES_FILE="${TESTS_FILE_PREFIX}-crashes.txt"
|
||||
UNIT_TESTS_FILE="${TESTS_FILE_PREFIX}.txt"
|
||||
|
||||
# Default to an empty known flakes file if it doesn't exist.
|
||||
assure_files ${FLAKES_FILE} ${FAILS_FILE} ${CRASHES_FILE}
|
||||
|
||||
# Remove unitTest_ prefix
|
||||
for UT_FILE in "${FAILS_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}"; do
|
||||
sed -i 's/^unitTest_//g' "${UT_FILE}"
|
||||
done
|
||||
|
||||
# create an exhaustive unittests list
|
||||
"${SKQP_BIN_DIR}"/list_gpu_unit_tests > "${GENERATED_FILE}"
|
||||
|
||||
# Remove undesirable tests from the list
|
||||
subtract_test_lists "${GENERATED_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}" "${FAILS_FILE}"
|
||||
|
||||
remove_comments_from_files "${GENERATED_FILE}"
|
||||
mv "${GENERATED_FILE}" "${UNIT_TESTS_FILE}"
|
||||
|
||||
echo "${UNIT_TESTS_FILE}"
|
||||
)
|
||||
|
||||
run_all_tests() {
|
||||
rm -f "${SKQP_ASSETS_DIR}"/skqp/*.txt
|
||||
}
|
||||
|
||||
copy_tests_files() (
|
||||
# Copy either unit test or render test files from a specific driver given by
|
||||
# GPU VERSION variable.
|
||||
# If there is no test file at the expected location, this function will
|
||||
# return error_code 1
|
||||
SKQP_BACKEND="${1}"
|
||||
SKQP_FILE_PREFIX="${INSTALL}/${GPU_VERSION}-skqp"
|
||||
|
||||
if echo "${SKQP_BACKEND}" | grep -qE 'vk|gl(es)?'
|
||||
then
|
||||
echo "Generating rendertests.txt file"
|
||||
GENERATED_RENDERTESTS=$(generate_rendertests)
|
||||
cp "${GENERATED_RENDERTESTS}" "${SKQP_ASSETS_DIR}"/skqp/rendertests.txt
|
||||
mkdir -p "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}"
|
||||
cp "${GENERATED_RENDERTESTS}" "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}/generated_rendertests.txt"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# The unittests.txt path is hardcoded inside assets directory,
|
||||
# that is why it needs to be a special case.
|
||||
if echo "${SKQP_BACKEND}" | grep -qE "unitTest"
|
||||
then
|
||||
echo "Generating unittests.txt file"
|
||||
GENERATED_UNITTESTS=$(generate_unittests)
|
||||
cp "${GENERATED_UNITTESTS}" "${SKQP_ASSETS_DIR}"/skqp/unittests.txt
|
||||
mkdir -p "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}"
|
||||
cp "${GENERATED_UNITTESTS}" "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}/generated_unittests.txt"
|
||||
fi
|
||||
)
|
||||
|
||||
resolve_tests_files() {
|
||||
if [ -n "${RUN_ALL_TESTS}" ]
|
||||
then
|
||||
run_all_tests
|
||||
return
|
||||
fi
|
||||
|
||||
SKQP_BACKEND=${1}
|
||||
if ! copy_tests_files "${SKQP_BACKEND}"
|
||||
then
|
||||
echo "No override test file found for ${SKQP_BACKEND}. Using the default one."
|
||||
fi
|
||||
}
|
||||
|
||||
test_vk_backend() {
|
||||
if echo "${SKQP_BACKENDS:?}" | grep -qE 'vk'
|
||||
then
|
||||
if [ -n "$VK_DRIVER" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "VK_DRIVER environment variable is missing."
|
||||
# shellcheck disable=SC2012
|
||||
VK_DRIVERS=$(ls "$INSTALL"/share/vulkan/icd.d/ | cut -f 1 -d '_')
|
||||
if [ -n "${VK_DRIVERS}" ]
|
||||
then
|
||||
echo "Please set VK_DRIVER to the correct driver from the list:"
|
||||
echo "${VK_DRIVERS}"
|
||||
fi
|
||||
echo "No Vulkan tests will be executed, but it was requested in SKQP_BACKENDS variable. Exiting."
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Vulkan environment is not configured, but it was not requested by the job
|
||||
return 1
|
||||
}
|
||||
|
||||
setup_backends() {
|
||||
if test_vk_backend
|
||||
then
|
||||
export VK_ICD_FILENAMES="$INSTALL"/share/vulkan/icd.d/"$VK_DRIVER"_icd."${VK_CPU:-$(uname -m)}".json
|
||||
fi
|
||||
}
|
||||
|
||||
show_reports() (
|
||||
set +xe
|
||||
|
||||
# Unit tests produce empty HTML reports, guide the user to check the TXT file.
|
||||
if echo "${SKQP_BACKENDS}" | grep -qE "unitTest"
|
||||
then
|
||||
# Remove the empty HTML report to avoid confusion
|
||||
rm -f "${SKQP_RESULTS_DIR}"/unitTest/report.html
|
||||
|
||||
echo "See skqp unit test results at:"
|
||||
echo "https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts${SKQP_RESULTS_DIR}/unitTest/unit_tests.txt"
|
||||
fi
|
||||
|
||||
REPORT_FILES=$(mktemp)
|
||||
find "${SKQP_RESULTS_DIR}"/**/report.html -type f > "${REPORT_FILES}"
|
||||
while read -r REPORT
|
||||
do
|
||||
# shellcheck disable=SC2001
|
||||
BACKEND_NAME=$(echo "${REPORT}" | sed 's@.*/\([^/]*\)/report.html@\1@')
|
||||
echo "See skqp ${BACKEND_NAME} render tests report at:"
|
||||
echo "https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts${REPORT}"
|
||||
done < "${REPORT_FILES}"
|
||||
|
||||
# If there is no report available, tell the user that something is wrong.
|
||||
if [ ! -s "${REPORT_FILES}" ]
|
||||
then
|
||||
echo "No skqp report available. Probably some fatal error has occured during the skqp execution."
|
||||
fi
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") [-a]
|
||||
|
||||
Arguments:
|
||||
-a: Run all unit tests and render tests, useful when introducing a new driver to skqp.
|
||||
EOF
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while getopts ':ah' opt; do
|
||||
case "$opt" in
|
||||
a)
|
||||
echo "Running all skqp tests"
|
||||
export RUN_ALL_TESTS=1
|
||||
shift
|
||||
;;
|
||||
|
||||
h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
|
||||
?)
|
||||
echo "Invalid command option."
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
set -e
|
||||
|
||||
parse_args "${@}"
|
||||
|
||||
# Needed so configuration files can contain paths to files in /install
|
||||
INSTALL="$CI_PROJECT_DIR"/install
|
||||
|
||||
if [ -z "$GPU_VERSION" ]; then
|
||||
echo 'GPU_VERSION must be set to something like "llvmpipe" or
|
||||
"freedreno-a630" (it will serve as a component to find the path for files
|
||||
residing in src/**/ci/*.txt)'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LD_LIBRARY_PATH=$INSTALL:$LD_LIBRARY_PATH
|
||||
setup_backends
|
||||
|
||||
SKQP_BIN_DIR=${SKQP_BIN_DIR:-/skqp}
|
||||
SKQP_ASSETS_DIR="${SKQP_BIN_DIR}"/assets
|
||||
SKQP_RESULTS_DIR="${SKQP_RESULTS_DIR:-${PWD}/results}"
|
||||
|
||||
mkdir -p "${SKQP_ASSETS_DIR}"/skqp
|
||||
|
||||
# Show the reports on exit, even when a test crashes
|
||||
trap show_reports INT TERM EXIT
|
||||
|
||||
SKQP_EXITCODE=0
|
||||
for SKQP_BACKEND in ${SKQP_BACKENDS}
|
||||
do
|
||||
resolve_tests_files "${SKQP_BACKEND}"
|
||||
SKQP_BACKEND_RESULTS_DIR="${SKQP_RESULTS_DIR}"/"${SKQP_BACKEND}"
|
||||
mkdir -p "${SKQP_BACKEND_RESULTS_DIR}"
|
||||
BACKEND_EXITCODE=0
|
||||
|
||||
GL_COLLAPSED=true gitlab_section_start "skqp_${SKQP_BACKEND}" "skqp logs for ${SKQP_BACKEND}"
|
||||
"${SKQP_BIN_DIR}"/skqp "${SKQP_ASSETS_DIR}" "${SKQP_BACKEND_RESULTS_DIR}" "${SKQP_BACKEND}_" ||
|
||||
BACKEND_EXITCODE=$?
|
||||
gitlab_section_end "skqp_${SKQP_BACKEND}"
|
||||
|
||||
if [ ! $BACKEND_EXITCODE -eq 0 ]
|
||||
then
|
||||
echo "skqp failed on ${SKQP_BACKEND} tests with exit code: ${BACKEND_EXITCODE}."
|
||||
else
|
||||
echo "skqp succeeded on ${SKQP_BACKEND}."
|
||||
fi
|
||||
|
||||
# Propagate error codes to leverage the final job result
|
||||
SKQP_EXITCODE=$(( SKQP_EXITCODE | BACKEND_EXITCODE ))
|
||||
done
|
||||
|
||||
exit $SKQP_EXITCODE
|
||||
|
|
@ -109,11 +109,6 @@ rustfmt:
|
|||
variables:
|
||||
DEQP_VER: vk
|
||||
|
||||
.skqp-test:
|
||||
variables:
|
||||
HWCI_START_XORG: 1
|
||||
HWCI_TEST_SCRIPT: "/install/skqp-runner.sh"
|
||||
|
||||
.fossilize-test:
|
||||
script:
|
||||
- ./install/fossilize-runner.sh
|
||||
|
|
|
|||
100
docs/ci/skqp.rst
100
docs/ci/skqp.rst
|
|
@ -8,94 +8,26 @@ device.
|
|||
|
||||
The rendering tests have support for GL, GLES and Vulkan backends and test some
|
||||
rendering scenarios.
|
||||
And the unit tests check the GPU behavior without rendering images.
|
||||
|
||||
Tests
|
||||
-----
|
||||
|
||||
Render tests design
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
It is worth noting that ``rendertests.txt`` can bring some detail about each test
|
||||
expectation, so each test can have a max pixel error count, to tell SkQP that it
|
||||
is OK to have at most that number of errors for that test. See also:
|
||||
https://github.com/google/skia/blob/c29454d1c9ebed4758a54a69798869fa2e7a36e0/tools/skqp/README_ALGORITHM.md
|
||||
|
||||
.. _test-location:
|
||||
|
||||
Location
|
||||
^^^^^^^^
|
||||
|
||||
Each ``rendertests.txt`` and ``unittest.txt`` file must be located inside a specific
|
||||
subdirectory inside SkQP assets directory.
|
||||
|
||||
+--------------+---------------------------------------------+
|
||||
| Test type | Location |
|
||||
+==============+=============================================+
|
||||
| Render tests | ``${SKQP_ASSETS_DIR}/skqp/rendertests.txt`` |
|
||||
+--------------+---------------------------------------------+
|
||||
| Unit tests | ``${SKQP_ASSETS_DIR}/skqp/unittests.txt`` |
|
||||
+--------------+---------------------------------------------+
|
||||
|
||||
The ``skqp-runner.sh`` script will make the necessary modifications to separate
|
||||
``rendertests.txt`` for each backend-driver combination. As long as the test files are located in the expected place:
|
||||
|
||||
+--------------+------------------------------------------------------------------------------------------------+
|
||||
| Test type | Location |
|
||||
+==============+================================================================================================+
|
||||
| Render tests | ``${MESA_REPOSITORY_DIR}/src/${GPU_DRIVER}/ci/${GPU_VERSION}-${SKQP_BACKEND}_rendertests.txt`` |
|
||||
+--------------+------------------------------------------------------------------------------------------------+
|
||||
| Unit tests | ``${MESA_REPOSITORY_DIR}/src/${GPU_DRIVER}/ci/${GPU_VERSION}_unittests.txt`` |
|
||||
+--------------+------------------------------------------------------------------------------------------------+
|
||||
|
||||
Where ``SKQP_BACKEND`` can be:
|
||||
|
||||
- gl: for GL backend
|
||||
- gles: for GLES backend
|
||||
- vk: for Vulkan backend
|
||||
|
||||
Example file
|
||||
""""""""""""
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
src/freedreno/ci/freedreno-a630-skqp-gl_rendertests.txt
|
||||
|
||||
- GPU_DRIVER: ``freedreno``
|
||||
- GPU_VERSION: ``freedreno-a630``
|
||||
- SKQP_BACKEND: ``gl``
|
||||
|
||||
.. _rendertests-design:
|
||||
And the unit tests check the GPU behavior without rendering images, using any of the GL/GLES or Vulkan drivers.
|
||||
|
||||
SkQP reports
|
||||
------------
|
||||
|
||||
SkQP generates reports after finishing its execution, they are located at the job
|
||||
artifacts results directory and are divided in subdirectories by rendering tests
|
||||
backends and unit
|
||||
tests. The job log has links to every generated report in order to facilitate
|
||||
the SkQP debugging.
|
||||
SkQP generates reports after finishing its execution, and deqp-runner collects
|
||||
them in the job artifacts results directory under the test name. Click the
|
||||
'Browse' button from a failing job to get to them.
|
||||
|
||||
Maintaining SkQP on Mesa CI
|
||||
---------------------------
|
||||
SkQP failing tests
|
||||
------------------
|
||||
|
||||
SkQP is built alongside with another binary, namely ``list_gpu_unit_tests``, it is
|
||||
located in the same folder where ``skqp`` binary is.
|
||||
SkQP rendering tests will have a range of pixel values allowed for the driver's
|
||||
rendering for a given test. This can make the "expected" image in the result
|
||||
output look rather strange, but you should be able to make sense of it knowing
|
||||
that.
|
||||
|
||||
This binary will generate the expected ``unittests.txt`` for the target GPU, so
|
||||
ideally it should be executed on every SkQP update and when a new device
|
||||
receives SkQP CI jobs.
|
||||
|
||||
1. Generate target unit tests for the current GPU with :code:`./list_gpu_unit_tests > unittests.txt`
|
||||
|
||||
2. Run SkQP job
|
||||
|
||||
3. If there is a failing or crashing unit test, remove it from the corresponding ``unittests.txt``
|
||||
|
||||
4. If there is a crashing render test, remove it from the corresponding ``rendertests.txt``
|
||||
|
||||
5. If there is a failing render test, visually inspect the result from the HTML report
|
||||
- If the render result is OK, update the max error count for that test
|
||||
- Otherwise, or put ``-1`` in the same threshold, as seen in :ref:`rendertests-design`
|
||||
|
||||
6. Remember to put the new tests files to the locations cited in :ref:`test-location`
|
||||
In SkQP itself, testcases can have increased failing pixel thresholds added to
|
||||
them to keep CI green when the rendering is "correct" but out of normal range.
|
||||
However, we don't support changing the thresholds in our testing. Because any
|
||||
driver rendering not meeting the normal thresholds will trigger Android CTS
|
||||
failures, we treat them as failures and track them as expected failures the
|
||||
```*-fails.txt`` file.`
|
||||
|
|
|
|||
Loading…
Reference in New Issue