gitlab-ci: Run the GLES2 CTS on llvmpipe.

This is the start of doing CTS tests on merges to Mesa master.  We use
the surfaceless platform so that we don't need to bother bringing up
weston or X11.  The surface size is kept low to reduce runtime, but
this comes at the cost of many rendering tests skipping due to
too-small render targets (as we see the impact of Mesa on the shared
runner pool, we can reevaluate this and what set of CTS tests we want
to run).

We split the job up across 4 runners (each at 4 llvmpipe threads), so
that the job can load-balance across our shared runners and finish
sooner (since dEQP is very single-thread-performance bound).

Reviewed-by: Eric Engestrom <eric.engestrom@intel.com>
This commit is contained in:
Eric Anholt 2019-06-28 16:35:32 -07:00
parent ab49873b44
commit 46daaca55e
5 changed files with 313 additions and 0 deletions

View File

@ -26,6 +26,7 @@ include:
stages:
- containers-build
- build+test
- test
# When to automatically run the CI
@ -39,6 +40,14 @@ stages:
when:
- runner_system_failure
.ci-deqp-artifacts: &ci-deqp-artifacts
artifacts:
when: always
untracked: false
paths:
# Watch out! Artifacts are relative to the build dir.
# https://gitlab.com/gitlab-org/gitlab-ce/commit/8788fb925706cad594adf6917a6c5f6587dd1521
- artifacts
# CONTAINERS
@ -77,6 +86,12 @@ debian:
- ccache --zero-stats || true
- ccache --show-stats || true
after_script:
# In case the install dir is being saved as artifacts, tar it up
# so that symlinks and hardlinks aren't each packed separately in
# the zip file.
- if [ -d install ]; then
tar -cf artifacts/install.tar install;
fi
- export CCACHE_DIR="$PWD/ccache"
- ccache --show-stats
@ -176,6 +191,7 @@ meson-main:
-D tools=all
MESON_SHADERDB: "true"
BUILDTYPE: "debugoptimized"
<<: *ci-deqp-artifacts
meson-clover:
extends: .meson-build
@ -247,6 +263,7 @@ meson-vulkan:
-D gallium-xa=false
-D gallium-nine=false
-D llvm=false
<<: *ci-deqp-artifacts
script:
- .gitlab-ci/meson-build.sh
@ -306,3 +323,32 @@ scons-llvm:
LLVM_VERSION: "3.4"
# LLVM 3.4 packages were built with an old libstdc++ ABI
CXX: "g++ -D_GLIBCXX_USE_CXX11_ABI=0"
.deqp-test:
<<: *ci-run-policy
stage: test
image: $DEBIAN_IMAGE
variables:
GIT_STRATEGY: none # testing doesn't build anything from source
DEQP_SKIPS: deqp-default-skips.txt
script:
# Note: Build dir (and thus install) may be dirty due to GIT_STRATEGY
- rm -rf install
- tar -xf artifacts/install.tar
- ./artifacts/deqp-runner.sh
artifacts:
when: on_failure
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths:
- results/
test-llvmpipe-gles2:
parallel: 4
variables:
DEQP_VER: gles2
DEQP_EXPECTED_FAILS: deqp-llvmpipe-fails.txt
LIBGL_ALWAYS_SOFTWARE: "true"
DEQP_RENDERER_MATCH: "llvmpipe"
extends: .deqp-test
dependencies:
- meson-main

View File

@ -0,0 +1,10 @@
# Note: skips lists for CI are just a list of lines that, when
# non-zero-length and not starting with '#', will regex match to
# delete lines from the test list. Be careful.
# Skip the perf/stress tests to keep runtime manageable
dEQP-GLES[0-9]*.performance
dEQP-GLES[0-9]*.stress
# These are really slow on tiling architectures (including llvmpipe).
dEQP-GLES[0-9]*.functional.flush_finish

View File

@ -0,0 +1,124 @@
dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center
dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner
dEQP-GLES2.functional.clipping.point.wide_point_clip
dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center
dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner
dEQP-GLES2.functional.clipping.triangle_vertex.clip_two.clip_neg_y_neg_z_and_neg_x_neg_y_pos_z
dEQP-GLES2.functional.clipping.triangle_vertex.clip_two.clip_pos_y_pos_z_and_neg_x_neg_y_neg_z
dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4
dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_depth_component16
dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_stencil_index8
dEQP-GLES2.functional.fbo.render.depth.rbo_rgba4_depth_component16
dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgba4
dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgba4_stencil_index8
dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgba4
dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgba4_stencil_index8
dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.no_rebind_rbo_rgba4_depth_component16
dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.rebind_rbo_rgba4_depth_component16
dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.no_rebind_rbo_rgba4_stencil_index8
dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.rebind_rbo_rgba4_stencil_index8
dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4
dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4_depth_component16
dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgba4_depth_component16
dEQP-GLES2.functional.polygon_offset.default_displacement_with_units
dEQP-GLES2.functional.polygon_offset.fixed16_displacement_with_units
dEQP-GLES2.functional.rasterization.interpolation.basic.line_loop_wide
dEQP-GLES2.functional.rasterization.interpolation.basic.line_strip_wide
dEQP-GLES2.functional.rasterization.interpolation.basic.lines_wide
dEQP-GLES2.functional.rasterization.interpolation.projected.line_loop_wide
dEQP-GLES2.functional.rasterization.interpolation.projected.line_strip_wide
dEQP-GLES2.functional.rasterization.interpolation.projected.lines_wide
dEQP-GLES2.functional.rasterization.limits.points
dEQP-GLES2.functional.shaders.texture_functions.fragment.texture2d_bias
dEQP-GLES2.functional.shaders.texture_functions.fragment.texture2dproj_vec3_bias
dEQP-GLES2.functional.shaders.texture_functions.fragment.texture2dproj_vec4_bias
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_clamp_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_mirror_etc1
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_mirror_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_repeat_etc1
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_repeat_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_clamp_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_mirror_etc1
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_mirror_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_etc1
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_l8
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_rgb888
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_rgba4444
dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_clamp_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_mirror_etc1
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_mirror_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_repeat_etc1
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_repeat_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_clamp_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_mirror_etc1
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_mirror_rgba8888
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_etc1
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_l8
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_rgb888
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_rgba4444
dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_rgba8888
dEQP-GLES2.functional.texture.mipmap.2d.affine.linear_linear_repeat
dEQP-GLES2.functional.texture.mipmap.2d.affine.nearest_linear_clamp
dEQP-GLES2.functional.texture.mipmap.2d.affine.nearest_linear_mirror
dEQP-GLES2.functional.texture.mipmap.2d.affine.nearest_linear_repeat
dEQP-GLES2.functional.texture.mipmap.2d.basic.linear_linear_repeat
dEQP-GLES2.functional.texture.mipmap.2d.basic.linear_linear_repeat_non_square
dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_clamp
dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_clamp_non_square
dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_mirror
dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_mirror_non_square
dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_repeat
dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_repeat_non_square
dEQP-GLES2.functional.texture.mipmap.2d.projected.linear_linear_repeat
dEQP-GLES2.functional.texture.mipmap.2d.projected.nearest_linear_clamp
dEQP-GLES2.functional.texture.mipmap.2d.projected.nearest_linear_mirror
dEQP-GLES2.functional.texture.mipmap.2d.projected.nearest_linear_repeat
dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_linear
dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_nearest
dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_linear
dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_nearest
dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_linear
dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_nearest
dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_linear_clamp
dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_linear_mirror
dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_linear_repeat
dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_nearest_clamp
dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_nearest_mirror
dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_nearest_repeat
dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_linear_clamp
dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_linear_mirror
dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_linear_repeat
dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_nearest_clamp
dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_nearest_mirror
dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_nearest_repeat
dEQP-GLES2.functional.texture.vertex.2d.wrap.clamp_clamp
dEQP-GLES2.functional.texture.vertex.2d.wrap.clamp_mirror
dEQP-GLES2.functional.texture.vertex.2d.wrap.clamp_repeat
dEQP-GLES2.functional.texture.vertex.2d.wrap.mirror_clamp
dEQP-GLES2.functional.texture.vertex.2d.wrap.mirror_mirror
dEQP-GLES2.functional.texture.vertex.2d.wrap.mirror_repeat
dEQP-GLES2.functional.texture.vertex.2d.wrap.repeat_clamp
dEQP-GLES2.functional.texture.vertex.2d.wrap.repeat_mirror
dEQP-GLES2.functional.texture.vertex.2d.wrap.repeat_repeat
dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_clamp
dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_mirror
dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_repeat
dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_clamp
dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_mirror
dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_repeat
dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_clamp
dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_mirror
dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_repeat
dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_nearest_clamp
dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_nearest_mirror
dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_nearest_repeat
dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_clamp
dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_mirror
dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_repeat
dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_clamp
dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_mirror
dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_repeat
dEQP-GLES2.functional.texture.vertex.cube.wrap.repeat_clamp
dEQP-GLES2.functional.texture.vertex.cube.wrap.repeat_mirror
dEQP-GLES2.functional.texture.vertex.cube.wrap.repeat_repeat

112
.gitlab-ci/deqp-runner.sh Executable file
View File

@ -0,0 +1,112 @@
#!/bin/bash
set -ex
DEQP_OPTIONS=(--deqp-surface-width=256 --deqp-surface-height=256)
DEQP_OPTIONS+=(--deqp-surface-type=pbuffer)
DEQP_OPTIONS+=(--deqp-gl-config-name=rgba8888d24s8ms0)
DEQP_OPTIONS+=(--deqp-visibility=hidden)
DEQP_OPTIONS+=(--deqp-log-images=disable)
DEQP_OPTIONS+=(--deqp-watchdog=enable)
DEQP_OPTIONS+=(--deqp-crashhandler=enable)
if [ -z "$DEQP_VER" ]; then
echo 'DEQP_VER must be set to something like "gles2" or "gles31" for the test run'
exit 1
fi
if [ -z "$DEQP_SKIPS" ]; then
echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"'
exit 1
fi
# Prep the expected failure list
if [ -n "$DEQP_EXPECTED_FAILS" ]; then
export DEQP_EXPECTED_FAILS=`pwd`/artifacts/$DEQP_EXPECTED_FAILS
else
export DEQP_EXPECTED_FAILS=/tmp/expect-no-failures.txt
touch $DEQP_EXPECTED_FAILS
fi
sort < $DEQP_EXPECTED_FAILS > /tmp/expected-fails.txt
# Fix relative paths on inputs.
export DEQP_SKIPS=`pwd`/artifacts/$DEQP_SKIPS
# Be a good citizen on the shared runners.
export LP_NUM_THREADS=4
# Set up the driver environment.
export LD_LIBRARY_PATH=`pwd`/install/lib/
export EGL_PLATFORM=surfaceless
# the runner was failing to look for libkms in /usr/local/lib for some reason
# I never figured out.
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
RESULTS=`pwd`/results
mkdir -p $RESULTS
cd /deqp/modules/$DEQP_VER
# Generate test case list file
cp /deqp/mustpass/$DEQP_VER-master.txt /tmp/case-list.txt
# Note: not using sorted input and comm, becuase I want to run the tests in
# the same order that dEQP would.
while read -r line; do
if echo "$line" | grep -q '^[^#]'; then
sed -i "/$line/d" /tmp/case-list.txt
fi
done < $DEQP_SKIPS
# If the job is parallel, take the corresponding fraction of the caselist.
# Note: N~M is a gnu sed extension to match every nth line (first line is #1).
if [ -n "$CI_NODE_INDEX" ]; then
sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
fi
if [ ! -s /tmp/case-list.txt ]; then
echo "Caselist generation failed"
exit 1
fi
# Cannot use tee because dash doesn't have pipefail
touch /tmp/result.txt
tail -f /tmp/result.txt &
./deqp-$DEQP_VER "${DEQP_OPTIONS[@]}" --deqp-log-filename=$RESULTS/results.qpa --deqp-caselist-file=/tmp/case-list.txt >> /tmp/result.txt
DEQP_EXITCODE=$?
sed -ne \
'/StatusCode="Fail"/{x;p}; s/#beginTestCaseResult //; T; h' \
$RESULTS/results.qpa \
> /tmp/unsorted-fails.txt
# Scrape out the renderer that the test run used, so we can validate that the
# right driver was used.
if grep -q "dEQP-.*.info.renderer" /tmp/case-list.txt; then
# This is an ugly dependency on the .qpa format: Print 3 lines after the
# match, which happens to contain the result.
RENDERER=`sed -n '/#beginTestCaseResult dEQP-.*.info.renderer/{n;n;n;p}' $RESULTS/results.qpa | sed -n -E "s|<Text>(.*)</Text>|\1|p"`
echo "GL_RENDERER for this test run: $RENDERER"
if [ -n "$DEQP_RENDERER_MATCH" ]; then
echo $RENDERER | grep -q $DEQP_RENDERER_MATCH > /dev/null
fi
fi
if [ $DEQP_EXITCODE -ne 0 ]; then
exit $DEQP_EXITCODE
fi
sort < /tmp/unsorted-fails.txt > $RESULTS/fails.txt
comm -23 $RESULTS/fails.txt /tmp/expected-fails.txt > /tmp/new-fails.txt
if [ -s /tmp/new-fails.txt ]; then
echo "Unexpected failures:"
cat /tmp/new-fails.txt
exit 1
else
echo "No new failures"
fi

View File

@ -39,3 +39,24 @@ cd ..
if test -n "$MESON_SHADERDB"; then
./.gitlab-ci/run-shader-db.sh;
fi
# Delete 2MB of includes from artifacts.
rm -rf install/include
# Strip the drivers in the artifacts to cut 80% of the artifacts size.
if [ -n "$CROSS" ]; then
STRIP=`sed -n -E "s/strip\s*=\s*'(.*)'/\1/p" /cross_file-$CROSS.txt`
if [ -z "$STRIP" ]; then
echo "Failed to find strip command in cross file"
exit 1
fi
else
STRIP="strip"
fi
find install -name \*.so -exec $STRIP {} \;
# Test runs don't pull down the git tree, so put the dEQP helper
# script and associated bits there.
mkdir -p artifacts/
cp -Rp .gitlab-ci/deqp* artifacts/
# cp -Rp src/freedreno/ci/expected* artifacts/