ci/deqp: Make sure that we pull in all board-specific xfail/skip/flake files.

When introducing/removing these files, it's easy to forget to update the yml to point to them. Instead of requiring the separate update, just have the runner script pick the right one from a single per-gpu variable. As a result, we now pick up the new deqp-lvp-skips.txt that was added but not conected. This also required moving some bypass flakes from the shared a630 flakes list to a separate list, which is a feature because now we'd notice the introduction of flakes to the gmem path. Fixes: ab79e6b8e3 ("ci: skip failing test on lavapipe") Acked-by: Daniel Stone <daniels@collabora.com> Reviewed-by: Juan A. Suarez <jasuarez@igalia.com> Reviewed-by: Christian Gmeiner <christian.gmeiner@gmail.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8147>
2020-12-17 11:08:42 -08:00 · 2020-12-17 11:08:42 -08:00 · 463dbbffa8
parent 9339ed2f85
commit 463dbbffa8
8 changed files with 40 additions and 51 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1063,8 +1063,6 @@ llvmpipe-piglit-quick_shader:
    PIGLIT_PROFILES: quick_shader

 .deqp-test:
-  variables:
-    DEQP_SKIPS: deqp-default-skips.txt
  script:
    - ./install/deqp-runner.sh

@ -1096,7 +1094,7 @@ llvmpipe-gles2:
    # Don't use threads inside llvmpipe, we've already got all cores
    # busy at the deqp-runner level.
    LP_NUM_THREADS: 0
-    DEQP_EXPECTED_FAILS: deqp-llvmpipe-fails.txt
+    GPU_VERSION: llvmpipe
    LIBGL_ALWAYS_SOFTWARE: "true"
    DEQP_EXPECTED_RENDERER: llvmpipe
  extends:
@ -1108,9 +1106,7 @@ softpipe-gles2:
    - llvmpipe-gles2
    - .softpipe-rules
  variables:
-    DEQP_EXPECTED_FAILS: deqp-softpipe-fails.txt
-    DEQP_SKIPS: deqp-softpipe-skips.txt
-    DEQP_FLAKES: deqp-softpipe-flakes.txt
+    GPU_VERSION: softpipe
    GALLIUM_DRIVER: "softpipe"
    DEQP_EXPECTED_RENDERER: softpipe

@ -1138,11 +1134,10 @@ virgl-gles2-on-gl:
  variables:
    DEQP_VER: gles2
    DEQP_NO_SAVE_RESULTS: 1
-    DEQP_SKIPS: deqp-virgl-gl-skips.txt
+    GPU_VERSION: virgl-gl
    # Don't use threads inside llvmpipe, we've already got all cores
    # busy at the deqp-runner level.
    LP_NUM_THREADS: 0
-    DEQP_EXPECTED_FAILS: deqp-virgl-gl-fails.txt
    DEQP_OPTIONS: "--deqp-log-images=disable"
    LIBGL_ALWAYS_SOFTWARE: "true"
    GALLIUM_DRIVER: "virpipe"
@ -1200,7 +1195,7 @@ virgl-gl32-on-gl:
 virgl-gles2-on-gles:
  variables:
    VIRGL_HOST_API: GLES
-    DEQP_EXPECTED_FAILS: deqp-virgl-gles-fails.txt
+    GPU_VERSION: virgl-gles
  extends:
    - virgl-gles2-on-gl
    - .test-manual
@ -1208,7 +1203,7 @@ virgl-gles2-on-gles:
 virgl-gles3-on-gles:
  variables:
    VIRGL_HOST_API: GLES
-    DEQP_EXPECTED_FAILS: deqp-virgl-gles-fails.txt
+    GPU_VERSION: virgl-gles
  extends:
    - virgl-gles3-on-gl
    - .test-manual
@ -1216,7 +1211,7 @@ virgl-gles3-on-gles:
 virgl-gles31-on-gles:
  variables:
    VIRGL_HOST_API: GLES
-    DEQP_EXPECTED_FAILS: deqp-virgl-gles-fails.txt
+    GPU_VERSION: virgl-gles
  extends:
    - virgl-gles31-on-gl
    - .test-manual
@ -1227,9 +1222,7 @@ arm64_a630_gles2:
  variables:
    BM_KERNEL: /lava-files/cheza-kernel
    BM_CMDLINE: "ip=dhcp console=ttyMSM0,115200n8 root=/dev/nfs rw nfsrootdebug nfsroot=,tcp,nfsvers=4.2 init=/init"
-    DEQP_EXPECTED_FAILS: deqp-freedreno-a630-fails.txt
-    DEQP_FLAKES: deqp-freedreno-a630-flakes.txt
-    DEQP_SKIPS: deqp-freedreno-a630-skips.txt
+    GPU_VERSION: freedreno-a630
    GIT_STRATEGY: none
    DEQP_EXPECTED_RENDERER: FD630
    VK_DRIVER: freedreno
@ -1297,7 +1290,7 @@ arm64_a630_vk_sysmem:
    DEQP_VER: vk
    DEQP_FRACTION: 15
    DEQP_CASELIST_FILTER: "dEQP-VK.renderpass.*"
-    DEQP_EXPECTED_FAILS: deqp-freedreno-a630-bypass-fails.txt
+    GPU_VERSION: freedreno-a630-bypass
    TU_DEBUG: sysmem

 .baremetal-test:
@ -1326,9 +1319,7 @@ arm64_a306_gles2:
    BM_CMDLINE: "ip=dhcp console=ttyMSM0,115200n8"
    FLAKES_CHANNEL: "#freedreno-ci"
    BARE_METAL_TEST_SCRIPT: "/install/deqp-runner.sh"
-    DEQP_EXPECTED_FAILS: deqp-freedreno-a307-fails.txt
-    DEQP_SKIPS: deqp-default-skips.txt
-    DEQP_FLAKES: deqp-freedreno-a307-flakes.txt
+    GPU_VERSION: freedreno-a307
    DEQP_PARALLEL: 0 # Default to number of CPUs
    DEQP_VER: gles2
    DEQP_EXPECTED_RENDERER: FD307
@ -1368,8 +1359,7 @@ arm64_a306_gles3_options:
    # Disable SMP because only CPU 0 is at a freq higher than 19mhz on
    # current upstream kernel.
    BM_CMDLINE: "ip=dhcp console=ttyMSM0,115200n8 nosmp"
-    DEQP_EXPECTED_FAILS: deqp-freedreno-a530-fails.txt
-    DEQP_FLAKES: deqp-freedreno-a530-flakes.txt
+    GPU_VERSION: freedreno-a530
    DEQP_EXPECTED_RENDERER: FD530
  tags:
    - google-freedreno-db820c
@ -1406,10 +1396,8 @@ vc4-rpi3-gles2:armhf:
    BM_KERNEL_MODULES: vc4
    BM_ROOTFS: /lava-files/rootfs-armhf
    BM_ROOTFS_EXTRA: /modules/armhf
-    DEQP_EXPECTED_FAILS: deqp-vc4-rpi3-fails.txt
+    GPU_VERSION: vc4-rpi3
    DEQP_EXPECTED_RENDERER: VC4
-    DEQP_FLAKES: deqp-vc4-rpi3-flakes.txt
-    DEQP_SKIPS: deqp-vc4-rpi3-skips.txt
    DEQP_VER: gles2
  script:
    - ./install/bare-metal/poe-powered.sh
@ -1425,9 +1413,7 @@ lavapipe-vk:
    - .llvmpipe-rules
    - .deqp-test-vk
  variables:
-    DEQP_EXPECTED_FAILS: deqp-lvp-fails.txt
-    DEQP_FLAKES: deqp-lvp-flakes.txt
-    DEQP_SKIPS: deqp-lvp-skips.txt
+    GPU_VERSION: lvp
    VK_DRIVER: lvp
    DEQP_FRACTION: 10

@ -1447,7 +1433,7 @@ radv_polaris10_vkcts:
    - .test-radv
    - .test-manual
  variables:
-    DEQP_SKIPS: deqp-radv-polaris10-skips.txt
+    GPU_VERSION: radv-polaris10
  tags:
    - polaris10

--- a/.gitlab-ci/bare-metal/arm64_a630_gles_others.sh
+++ b/.gitlab-ci/bare-metal/arm64_a630_gles_others.sh
@ -23,7 +23,7 @@ if ! env \
  DEQP_VER=gles31 \
  DEQP_FRACTION=5 \
  FD_MESA_DEBUG=nogmem \
-  DEQP_EXPECTED_FAILS=deqp-freedreno-a630-bypass-fails.txt \
+  GPU_VERSION=freedreno-a630-bypass \
  /install/deqp-runner.sh; then
    EXIT=1
 fi
--- a/.gitlab-ci/bare-metal/rootfs-setup.sh
+++ b/.gitlab-ci/bare-metal/rootfs-setup.sh
@ -30,15 +30,12 @@ for var in \
    CI_RUNNER_DESCRIPTION \
    DEQP_CASELIST_FILTER \
    DEQP_CONFIG \
-    DEQP_EXPECTED_FAILS \
    DEQP_EXPECTED_RENDERER \
    DEQP_FRACTION \
    DEQP_HEIGHT \
    DEQP_NO_SAVE_RESULTS \
-    DEQP_FLAKES \
    DEQP_PARALLEL \
    DEQP_RESULTS_DIR \
-    DEQP_SKIPS \
    DEQP_VARIANT \
    DEQP_VER \
    DEQP_WIDTH \
@ -46,6 +43,7 @@ for var in \
    DRIVER_NAME \
    FD_MESA_DEBUG \
    FLAKES_CHANNEL \
+    GPU_VERSION \
    IR3_SHADER_DEBUG \
    MESA_GL_VERSION_OVERRIDE \
    MESA_GLSL_VERSION_OVERRIDE \
--- a/.gitlab-ci/deqp-freedreno-a630-bypass-flakes.txt
+++ b/.gitlab-ci/deqp-freedreno-a630-bypass-flakes.txt
@ -0,0 +1,11 @@
+# Note: flakes lists for CI are just a list of lines that, when
+# non-zero-length and not starting with '#', will regex match to
+# delete lines from the test list.  Be careful.
+
+# At least some of the separate_channels tests fail on sysmem due to an
+# interaction of use of a UBWC buffer as both a render target and a
+# texture.  Stores are done through both paths in separate channels,
+# and the UBWC updates don't get synced.  The current a650 blob also
+# fails these tests and qcom apparently noted the failure at one point
+# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2017
+dEQP-VK.renderpass.*separate_channels.*
--- a/.gitlab-ci/deqp-freedreno-a630-flakes.txt
+++ b/.gitlab-ci/deqp-freedreno-a630-flakes.txt
@ -28,14 +28,6 @@ dEQP-VK.pipeline.spec_constant.compute.composite.matrix.mat3x2
 # Fails NIR_VALIDATE so probably flaky
 dEQP-VK.memory_model.write_after_read.core11.u32.coherent.fence_fence.atomicwrite.workgroup.payload_nonlocal.workgroup.guard_local.buffer.comp

-# At least some of the separate_channels tests fail on sysmem due to an
-# interaction of use of a UBWC buffer as both a render target and a
-# texture.  Stores are done through both paths in separate channels,
-# and the UBWC updates don't get synced.  The current a650 blob also
-# fails these tests and qcom apparently noted the failure at one point
-# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2017
-dEQP-VK.renderpass.*separate_channels.*
-
 # Undiagnosed flakes appearing more than once in the last 2 months as
 # of 2020-08-19, in descending order of frequency.
 dEQP-GLES3.functional.fbo.msaa.2_samples.stencil_index8
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@ -24,8 +24,8 @@ if [ "$DEQP_VER" = "vk" ]; then
   fi
 fi

-if [ -z "$DEQP_SKIPS" ]; then
-   echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"'
+if [ -z "$GPU_VERSION" ]; then
+   echo 'GPU_VERSION must be set to something like "llvmpipe" or "freedreno-a630" (the name used in .gitlab-ci/deqp-gpu-version-*.txt)'
   exit 1
 fi

@ -79,12 +79,18 @@ if [ ! -s /tmp/case-list.txt ]; then
    exit 1
 fi

-if [ -n "$DEQP_EXPECTED_FAILS" ]; then
-    BASELINE="--baseline $INSTALL/$DEQP_EXPECTED_FAILS"
+if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then
+    DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --baseline $INSTALL/deqp-$GPU_VERSION-fails.txt"
 fi

-if [ -n "$DEQP_FLAKES" ]; then
-    FLAKES="--flakes $INSTALL/$DEQP_FLAKES"
+if [ -e "$INSTALL/deqp-$GPU_VERSION-flakes.txt" ]; then
+    DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt"
+fi
+
+if [ -e "$INSTALL/deqp-$GPU_VERSION-skips.txt" ]; then
+    DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --skips $INSTALL/deqp-$GPU_VERSION-skips.txt"
+else
+    DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --skips $INSTALL/deqp-default-skips.txt"
 fi

 set +e
@ -114,9 +120,6 @@ run_cts() {
        --deqp $deqp \
        --output $RESULTS \
        --caselist $caselist \
-        --skips $INSTALL/$DEQP_SKIPS \
-        $BASELINE \
-        $FLAKES \
        $JOB \
        $SUMMARY_LIMIT \
 	$DEQP_RUNNER_OPTIONS \
--- a/.gitlab-ci/lava-deqp.yml.jinja2
+++ b/.gitlab-ci/lava-deqp.yml.jinja2
@ -95,8 +95,7 @@ actions:
          - wget -S --progress=dot:giga -O- {{ mesa_url }} | tar -xz

          - export DEQP_NO_SAVE_RESULTS=1
-          - export DEQP_EXPECTED_FAILS=deqp-{{ gpu_version }}-fails.txt
-          - export DEQP_SKIPS=deqp-{{ gpu_version }}-skips.txt
+          - export GPU_VERSION={{ gpu_version }}
          - export DEQP_VER={{ deqp_version }}
          - export LIBGL_DRIVERS_PATH=`pwd`/install/lib/dri

--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@ -170,7 +170,7 @@ panfrost-t720-gles2:arm64:
    DEVICE_TYPE: sun50i-h6-pine-h64
    DTB: ${DEVICE_TYPE}
    GPU_VERSION: panfrost-t720
-    ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T720 DEQP_FLAKES=deqp-panfrost-t720-flakes.txt"
+    ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T720"
  tags:
    - mesa-ci-aarch64-lava-collabora