virgl/ci: Run each dEQP instance in its own VM

Currently we run deqp-runner inside a single VM, which makes very poor use of the available CPUs because Virgl has a bottleneck in the VMM that serializes everything. With this change, we can run several Crosvm instances in a runner and make full use of the CPUs. Getting the same coverage with 3 runners instead of 6. Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com> Reviewed-by: Emma Anholt <emma@anholt.net> Reviewed-by: Corentin Noël <corentin.noel@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12828>
2021-09-08 07:52:58 +02:00 · 2021-09-08 07:52:58 +02:00 · 81f25d8f27
parent 4bfcbe3f69
commit 81f25d8f27
12 changed files with 140 additions and 181 deletions
--- a/.gitlab-ci/crosvm-init.sh
+++ b/.gitlab-ci/crosvm-init.sh
@ -1,27 +1,29 @@
 #!/bin/sh

-set -ex
+set -e
+
+export DEQP_TEMP_DIR=$1

 mount -t proc none /proc
 mount -t sysfs none /sys
-mount -t devtmpfs none /dev || echo possibly already mounted
 mkdir -p /dev/pts
 mount -t devpts devpts /dev/pts
 mount -t tmpfs tmpfs /tmp

-. /crosvm-env.sh
+. $DEQP_TEMP_DIR/crosvm-env.sh

-# / is ro
-export PIGLIT_REPLAY_EXTRA_ARGS="$PIGLIT_REPLAY_EXTRA_ARGS --db-path /tmp/replayer-db"
+cd $PWD

-if sh $CROSVM_TEST_SCRIPT; then
-    touch /results/success
-fi
+dmesg --level crit,err,warn -w >> $DEQP_TEMP_DIR/stderr &

-sleep 5   # Leave some time to get the last output flushed out
+set +e
+stdbuf -oL sh $DEQP_TEMP_DIR/crosvm-script.sh 2>> $DEQP_TEMP_DIR/stderr >> $DEQP_TEMP_DIR/stdout
+echo $? > $DEQP_TEMP_DIR/exit_code
+set -e
+
+sync
+sleep 1

 poweroff -d -n -f || true

-sleep 10   # Just in case init would exit before the kernel shuts down the VM
-
-exit 1
+sleep 1   # Just in case init would exit before the kernel shuts down the VM
--- a/.gitlab-ci/crosvm-runner.sh
+++ b/.gitlab-ci/crosvm-runner.sh
@ -1,25 +1,20 @@
 #!/bin/sh

-set -x
+set -e

-ln -sf $CI_PROJECT_DIR/install /install
+# This script can be called concurrently, pass arguments and env in a per-instance tmp dir
+export DEQP_TEMP_DIR=`mktemp -d /tmp.XXXXXXXXXX`

-export LD_LIBRARY_PATH=$CI_PROJECT_DIR/install/lib/
-export EGL_PLATFORM=surfaceless
+# The dEQP binary needs to run from the directory it's in
+if [ -z "${1##*"deqp"*}" ]; then
+  PWD=`dirname $1`
+fi

-export -p > /crosvm-env.sh
-export GALLIUM_DRIVER="$CROSVM_GALLIUM_DRIVER"
-export GALLIVM_PERF="nopt"
-export LIBGL_ALWAYS_SOFTWARE="true"
+export -p > $DEQP_TEMP_DIR/crosvm-env.sh

-CROSVM_KERNEL_ARGS="root=my_root rw rootfstype=virtiofs loglevel=3 init=$CI_PROJECT_DIR/install/crosvm-init.sh ip=192.168.30.2::192.168.30.1:255.255.255.0:crosvm:eth0"
+CROSVM_KERNEL_ARGS="console=null root=my_root rw rootfstype=virtiofs init=$CI_PROJECT_DIR/install/crosvm-init.sh ip=192.168.30.2::192.168.30.1:255.255.255.0:crosvm:eth0 -- $DEQP_TEMP_DIR"

-# Temporary results dir because from the guest we cannot write to /
-mkdir -p /results
-mount -t tmpfs tmpfs /results
-
-mkdir -p /piglit/.gitlab-ci/piglit
-mount -t tmpfs tmpfs /piglit/.gitlab-ci/piglit
+echo $@ > $DEQP_TEMP_DIR/crosvm-script.sh

 unset DISPLAY
 unset XDG_RUNTIME_DIR
@ -27,20 +22,25 @@ unset XDG_RUNTIME_DIR
 /usr/sbin/iptables-legacy  -t nat -A POSTROUTING -o eth0 -j MASQUERADE
 echo 1 > /proc/sys/net/ipv4/ip_forward

-# We aren't testing LLVMPipe here, so we don't need to validate NIR on the host
-export NIR_VALIDATE=0
+# Send output from guest to host
+touch $DEQP_TEMP_DIR/stderr $DEQP_TEMP_DIR/stdout
+tail -f $DEQP_TEMP_DIR/stderr > /dev/stderr &
+ERR_TAIL_PID=$!
+tail -f $DEQP_TEMP_DIR/stdout > /dev/stdout &
+OUT_TAIL_PID=$!

-crosvm run \
+trap "exit \$exit_code" INT TERM
+trap "exit_code=\$?; kill $ERR_TAIL_PID $OUT_TAIL_PID" EXIT
+
+# We aren't testing LLVMPipe here, so we don't need to validate NIR on the host
+NIR_VALIDATE=0 LIBGL_ALWAYS_SOFTWARE="true" GALLIUM_DRIVER="$CROSVM_GALLIUM_DRIVER" stdbuf -oL crosvm run \
  --gpu "$CROSVM_GPU_ARGS" \
  -m 4096 \
-  -c $((FDO_CI_CONCURRENT > 1 ? FDO_CI_CONCURRENT - 1 : 1)) \
+  -c 2 \
  --disable-sandbox \
  --shared-dir /:my_root:type=fs:writeback=true:timeout=60:cache=always \
  --host_ip=192.168.30.1 --netmask=255.255.255.0 --mac "AA:BB:CC:00:00:12" \
  -p "$CROSVM_KERNEL_ARGS" \
-  /lava-files/bzImage
+  /lava-files/bzImage >> $DEQP_TEMP_DIR/stderr > /dev/null

-mkdir -p $CI_PROJECT_DIR/results
-mv /results/* $CI_PROJECT_DIR/results/.
-
-test -f $CI_PROJECT_DIR/results/success
+exit `cat $DEQP_TEMP_DIR/exit_code`
--- a/src/gallium/drivers/virgl/ci/deqp-virgl-gl.toml
+++ b/src/gallium/drivers/virgl/ci/deqp-virgl-gl.toml
@ -1,51 +1,57 @@
 [[deqp]]
-deqp = "/deqp/modules/gles2/deqp-gles2"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles2-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles2/deqp-gles2",
    "--deqp-surface-width=256",
    "--deqp-surface-height=256",
    "--deqp-surface-type=pbuffer",
    "--deqp-gl-config-name=rgba8888d24s8ms0",
    "--deqp-visibility=hidden"
 ]
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
 version_check = "GL ES 3.2.*git"
-renderer_check = "virgl"
+renderer_check = "virgl.*llvmpipe"

 [[deqp]]
-deqp = "/deqp/modules/gles3/deqp-gles3"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles3-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles3/deqp-gles3",
    "--deqp-surface-width=256",
    "--deqp-surface-height=256",
    "--deqp-surface-type=pbuffer",
    "--deqp-gl-config-name=rgba8888d24s8ms0",
    "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time

 [[deqp]]
-deqp = "/deqp/modules/gles31/deqp-gles31"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles31-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles31/deqp-gles31",
    "--deqp-surface-width=256",
    "--deqp-surface-height=256",
    "--deqp-surface-type=pbuffer",
    "--deqp-gl-config-name=rgba8888d24s8ms0",
    "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time

 [[deqp]]
-deqp = "/deqp/external/openglcts/modules/glcts"
+deqp = "/install/crosvm-runner.sh"
 caselists = [
    "/deqp/mustpass/gl30-master.txt",
    "/deqp/mustpass/gl31-master.txt",
    "/deqp/mustpass/gl32-master.txt",
 ]
 deqp_args = [
+    "/deqp/external/openglcts/modules/glcts",
    "--deqp-surface-width=256",
    "--deqp-surface-height=256",
    "--deqp-surface-type=pbuffer",
    "--deqp-gl-config-name=rgba8888d24s8ms0",
    "--deqp-visibility=hidden"
 ]
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
--- a/src/gallium/drivers/virgl/ci/deqp-virgl-gles.toml
+++ b/src/gallium/drivers/virgl/ci/deqp-virgl-gles.toml
@ -1,36 +1,57 @@
 [[deqp]]
-deqp = "/deqp/modules/gles2/deqp-gles2"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles2-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles2/deqp-gles2",
    "--deqp-surface-width=256",
    "--deqp-surface-height=256",
    "--deqp-surface-type=pbuffer",
    "--deqp-gl-config-name=rgba8888d24s8ms0",
    "--deqp-visibility=hidden"
 ]
-version_check = "GL ES 3.1.*git"
-renderer_check = "virgl"
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
+version_check = "GL ES 3.2.*git"
+renderer_check = "virgl.*llvmpipe"

 [[deqp]]
-deqp = "/deqp/modules/gles3/deqp-gles3"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles3-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles3/deqp-gles3",
    "--deqp-surface-width=256",
    "--deqp-surface-height=256",
    "--deqp-surface-type=pbuffer",
    "--deqp-gl-config-name=rgba8888d24s8ms0",
    "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time

 [[deqp]]
-deqp = "/deqp/modules/gles31/deqp-gles31"
+deqp = "/install/crosvm-runner.sh"
 caselists = ["/deqp/mustpass/gles31-master.txt"]
 deqp_args = [
+    "/deqp/modules/gles31/deqp-gles31",
    "--deqp-surface-width=256",
    "--deqp-surface-height=256",
    "--deqp-surface-type=pbuffer",
    "--deqp-gl-config-name=rgba8888d24s8ms0",
    "--deqp-visibility=hidden"
 ]
-timeout = 180.0
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
+
+[[deqp]]
+deqp = "/install/crosvm-runner.sh"
+caselists = [
+    "/deqp/mustpass/gl30-master.txt",
+    "/deqp/mustpass/gl31-master.txt",
+    "/deqp/mustpass/gl32-master.txt",
+]
+deqp_args = [
+    "/deqp/external/openglcts/modules/glcts",
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+timeout = 360.0 # Starting 8 Crosvm instances simultaneously can take some time
--- a/src/gallium/drivers/virgl/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/virgl/ci/gitlab-ci.yml
@ -5,22 +5,17 @@
  variables:
    GALLIUM_DRIVER: "virgl"
    CROSVM_GALLIUM_DRIVER: "llvmpipe"
-    CROSVM_GPU_ARGS: "gles=false,backend=virglrenderer,egl=true,surfaceless=true"
+    CROSVM_GPU_ARGS: "gles=false,backend=virglrenderer,egl=true,surfaceless=true,width=1024,height=768"
    GALLIVM_PERF: "nopt,no_quad_lod"

 virgl-on-gl:
  variables:
    DEQP_SUITE: virgl-gl
    GPU_VERSION: virgl-gl
-    # Speed rendering a bit, as crosvm is processing requests serially
-    LP_NUM_THREADS: 8
-    FDO_CI_CONCURRENT: 3
-    CROSVM_TEST_SCRIPT: "/install/deqp-runner.sh"
-  parallel: 6
+    LP_NUM_THREADS: 1 # There will be FDO_CI_CONCURRENT Crosvm processes, so each should use a single thread
+  parallel: 3
  tags:
    - kvm
-  script:
-    - install/crosvm-runner.sh
  extends:
    - .deqp-test
    - .virgl-test
@ -30,10 +25,9 @@ virgl-on-gles:
    VIRGL_HOST_API: GLES
    DEQP_SUITE: virgl-gles
    GPU_VERSION: virgl-gles
-    CROSVM_GPU_ARGS: "gles=true,backend=virglrenderer,egl=true,surfaceless=true"
+    CROSVM_GPU_ARGS: "gles=true,backend=virglrenderer,egl=true,surfaceless=true,width=1024,height=768"
  extends:
    - virgl-on-gl
-    - .test-manual

 virgl-traces:
  extends:
@ -44,9 +38,10 @@ virgl-traces:
    PIGLIT_REPLAY_DESCRIPTION_FILE: "${CI_PROJECT_DIR}/install/traces-virgl.yml"
    PIGLIT_REPLAY_DEVICE_NAME: "gl-virgl"
    PIGLIT_RESULTS: "virgl-replay"
-    CROSVM_TEST_SCRIPT: "/install/piglit/run.sh"
+    LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/install/lib/"     # For Crosvm
  tags:
    - kvm
  script:
-    - install/crosvm-runner.sh
-
+    # Use all threads for rendering and only run one job at a time
+    # Couldn't get GitLab CI to correctly substitute the variable in the yaml
+    - LP_NUM_THREADS=${FDO_CI_CONCURRENT} FDO_CI_CONCURRENT=1 install/crosvm-runner.sh install/piglit/run.sh
--- a/src/gallium/drivers/virgl/ci/traces-virgl.yml
+++ b/src/gallium/drivers/virgl/ci/traces-virgl.yml
@ -231,7 +231,7 @@ traces:
  - path: supertuxkart/supertuxkart-mansion-egl-gles.trace
    expectations:
      - device: gl-virgl
-        checksum: 156c26de2cefe1973b1593e6b22f7edb
+        checksum: c8e5d7c4377b8e484ae41270692914f3
  - path: xonotic/xonotic-keybench-high.trace
    expectations:
      - device: gl-virgl
--- a/src/gallium/drivers/virgl/ci/virgl-gl-fails.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gl-fails.txt
@ -1,8 +1,6 @@
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner,Fail
 dEQP-GLES2.functional.clipping.point.wide_point_clip,Fail
-dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center,Fail
-dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_corner,Fail
 dEQP-GLES3.functional.clipping.point.wide_point_clip,Fail
@ -20,8 +18,7 @@ dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_dst_y,Fa
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_x,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.4,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_implementation_draw_buffers.8,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_12,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_writeonly_12,Fail
+dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.9,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_equal,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_larger,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.fbo_bbox_equal,Fail
@ -40,6 +37,7 @@ dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_rbo_2,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_1,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_2,Fail
+dEQP-GLES31.functional.ssbo.layout.random.all_shared_buffer.36,Fail
 # https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2892
 KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x2_vs,Crash
 KHR-GL30.transform_feedback.api_errors_test,Fail
--- a/src/gallium/drivers/virgl/ci/virgl-gl-flakes.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gl-flakes.txt
@ -4,3 +4,14 @@ KHR-GL30.shaders30.glsl_constructors.bvec4_from_bool_mat3_vs
 KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x3_vs
 # https://gitlab.freedesktop.org/mesa/mesa/-/issues/4651
 KHR-GL31.transform_feedback.capture_special_interleaved_test
+
+dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_12
+dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_12
+
+dEQP-GLES2.functional.clipping.point.wide_point_clip
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner
+
+dEQP-GLES3.functional.clipping.point.wide_point_clip
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_corner
--- a/src/gallium/drivers/virgl/ci/virgl-gl-skips.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gl-skips.txt
@ -1,2 +1,6 @@
+# Note: skips lists for CI are just a list of lines that, when
+# non-zero-length and not starting with '#', will regex match to
+# delete lines from the test list.  Be careful.
+
 # Sometimes crashes, e.g. https://gitlab.freedesktop.org/kusma/mesa/-/jobs/4109419
 dEQP-GLES31.functional.compute.basic.empty
--- a/src/gallium/drivers/virgl/ci/virgl-gles-fails.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gles-fails.txt
@ -1,13 +1,10 @@
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner,Fail
-dEQP-GLES2.functional.clipping.point.wide_point_clip,Fail
 dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center,Fail
 dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner,Fail
-dEQP-GLES2.functional.polygon_offset.default_displacement_with_units,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_center,Fail
 dEQP-GLES3.functional.clipping.line.wide_line_clip_viewport_corner,Fail
 dEQP-GLES3.functional.clipping.point.wide_point_clip,Fail
-dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_center,Fail
 dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_corner,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_mag,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_mag_reverse_dst_x,Fail
@ -19,13 +16,9 @@ dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_dst_x,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_dst_x,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_dst_y,Fail
 dEQP-GLES3.functional.fbo.blit.rect.nearest_consistency_min_reverse_src_x,Fail
-dEQP-GLES3.functional.polygon_offset.default_displacement_with_units,Fail
-dEQP-GLES3.functional.polygon_offset.fixed24_displacement_with_units,Fail
-dEQP-GLES3.functional.polygon_offset.float32_displacement_with_units,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.4,Fail
+dEQP-GLES31.functional.draw_buffers_indexed.random.max_implementation_draw_buffers.8,Fail
 dEQP-GLES31.functional.draw_buffers_indexed.random.max_required_draw_buffers.9,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_12,Fail
-dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_32,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_equal,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.default_framebuffer_bbox_larger,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.global_state.vertex_tessellation_fragment.fbo_bbox_equal,Fail
@ -36,110 +29,6 @@ dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_d
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_draw.vertex_tessellation_fragment.fbo_bbox_larger,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_primitive.vertex_tessellation_fragment.default_framebuffer,Fail
 dEQP-GLES31.functional.primitive_bounding_box.wide_points.tessellation_set_per_primitive.vertex_tessellation_fragment.fbo,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_atomic_counter_bindings,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_atomic_counter_buffer_size,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_shader_output_resources,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_combined_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_uniform_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_work_group_count,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_compute_work_group_size,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_draw_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_input_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_fragment_uniform_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_program_texel_offset,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_output_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.max_vertex_uniform_vectors,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.core.min_program_texel_offset,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_input_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_output_vertices,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_total_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.geometry_shader.max_geometry_uniform_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.sample_variables.max_samples,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_patch_vertices,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_input_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_control_total_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_atomic_counter_buffers,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_atomic_counters,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_image_uniforms,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_input_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_output_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_texture_image_units,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_evaluation_uniform_components,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_gen_level,Fail
-dEQP-GLES31.functional.shaders.builtin_constants.tessellation_shader.max_tess_patch_components,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.atomic_counter.const_expression_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.atomic_counter.dynamically_uniform_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.atomic_counter.uniform_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.sampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_expression.tessellation_control.usamplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.const_literal.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.isamplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.sampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.dynamically_uniform.tessellation_control.usamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.isampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.isampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.isamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.sampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.sampler2dshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.sampler3d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.samplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.samplercube,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.samplercubeshadow,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.usampler2darray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.usampler2d,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.sampler.uniform.tessellation_control.usamplercubearray,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ssbo.const_expression_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ssbo.const_literal_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ubo.const_expression_tessellation_control,Fail
-dEQP-GLES31.functional.shaders.opaque_type_indexing.ubo.dynamically_uniform_tessellation_control,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_pixel.multisample_rbo_1,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_pixel.multisample_rbo_2,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_pixel.multisample_texture_1,Fail
@ -148,6 +37,11 @@ dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_rbo_2,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_1,Fail
 dEQP-GLES31.functional.shaders.sample_variables.sample_mask_in.bit_count_per_two_samples.multisample_texture_2,Fail
+dEQP-GLES31.functional.ssbo.layout.random.all_shared_buffer.36,Fail
+dEQP-GLES31.functional.ssbo.layout.random.nested_structs_arrays_instance_arrays.22,Fail
+KHR-GL30.glsl_noperspective.functionaltest,Fail
+# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2892
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x2_vs,Crash
 KHR-GL30.transform_feedback.api_errors_test,Fail
 KHR-GL30.transform_feedback.capture_vertex_interleaved_test,Fail
 KHR-GL30.transform_feedback.capture_vertex_separate_test,Fail
@ -158,13 +52,15 @@ KHR-GL30.transform_feedback.get_xfb_varying,Fail
 KHR-GL30.transform_feedback.query_vertex_interleaved_test,Fail
 KHR-GL30.transform_feedback.query_vertex_separate_test,Fail
 KHR-GL31.CommonBugs.CommonBug_ParenthesisInLayoutQualifierIntegerValue,Fail
+KHR-GL31.transform_feedback.capture_special_interleaved_test,Crash
 KHR-GL31.transform_feedback.capture_vertex_interleaved_test,Fail
 KHR-GL31.transform_feedback.capture_vertex_separate_test,Fail
 KHR-GL31.transform_feedback.discard_vertex_test,Fail
-KHR-GL31.transform_feedback.draw_xfb_instanced_test,Fail
+KHR-GL31.transform_feedback.draw_xfb_instanced_test,Crash
 KHR-GL31.transform_feedback.draw_xfb_stream_instanced_test,Crash
 KHR-GL31.transform_feedback.query_vertex_interleaved_test,Fail
 KHR-GL31.transform_feedback.query_vertex_separate_test,Fail
+KHR-GL31.glsl_noperspective.functionaltest,Fail
 KHR-GL32.CommonBugs.CommonBug_ParenthesisInLayoutQualifierIntegerValue,Fail
 KHR-GL32.transform_feedback.capture_vertex_interleaved_test,Fail
 KHR-GL32.transform_feedback.capture_vertex_separate_test,Fail
@ -179,3 +75,4 @@ KHR-GL32.transform_feedback_overflow_query_ARB.multiple-streams-multiple-buffers
 KHR-GL32.transform_feedback_overflow_query_ARB.multiple-streams-one-buffer-per-stream,Fail
 KHR-GL32.transform_feedback.query_vertex_interleaved_test,Fail
 KHR-GL32.transform_feedback.query_vertex_separate_test,Fail
+KHR-GL32.glsl_noperspective.functionaltest,Fail
--- a/src/gallium/drivers/virgl/ci/virgl-gles-flakes.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gles-flakes.txt
@ -0,0 +1,17 @@
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x2_vs
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_bool_mat4x3_vs
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_bool_mat3_vs
+KHR-GL30.shaders30.glsl_constructors.bvec4_from_mat4x3_vs
+# https://gitlab.freedesktop.org/mesa/mesa/-/issues/4651
+KHR-GL31.transform_feedback.capture_special_interleaved_test
+
+dEQP-GLES31.functional.image_load_store.buffer.image_size.writeonly_12
+dEQP-GLES31.functional.image_load_store.buffer.image_size.readonly_12
+
+dEQP-GLES2.functional.clipping.point.wide_point_clip
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner
+
+dEQP-GLES3.functional.clipping.point.wide_point_clip
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES3.functional.clipping.point.wide_point_clip_viewport_corner
--- a/src/gallium/drivers/virgl/ci/virgl-gles-skips.txt
+++ b/src/gallium/drivers/virgl/ci/virgl-gles-skips.txt
@ -0,0 +1,8 @@
+# Note: skips lists for CI are just a list of lines that, when
+# non-zero-length and not starting with '#', will regex match to
+# delete lines from the test list.  Be careful.
+
+KHR-GL32.packed_pixels.varied_rectangle.depth*
+
+# Sometimes crashes, e.g. https://gitlab.freedesktop.org/kusma/mesa/-/jobs/4109419
+dEQP-GLES31.functional.compute.basic.empty