ci/deqp: Upgrade the runner, enable junit output.

I moved QPA-to-XML conversion to the runner, so Mesa CI (and developers!) don't need to do quite so much in bash. I also made it clean up caselist .qpa files since nobody ever wants them and we deleted them anyway. This cleans up a ton of the job log output. Additionally, I added a subcommend to turn the .csv into a junit output that we can expose to gitlab. Now, the pipeline's status page will report the failed testcases, and the "detail" button will give you a link to the .XML to view for the failure. (We don't report all testcases because it's too much load for the gitlab server). Note that this will 404 for the LAVA runners for now, as they don't retain artifacts in gitlab (the plan is to eventually have them minio upload the artifacts). This uprev also includes a deqp output parsing fix, resulting in us catching a couple more failures in some drivers. Reviewed-by: Christian Gmeiner <christian.gmeiner@gmail.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8206>
2020-12-18 16:47:51 -08:00 · 2020-12-18 16:47:51 -08:00 · 302b4f7724
parent 35e9143428
commit 302b4f7724
9 changed files with 43 additions and 49 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -362,13 +362,13 @@ x86_test-base:
 x86_test-gl:
  extends: .use-x86_test-base
  variables:
-    MESA_IMAGE_TAG: &x86_test-gl "2020-12-18-piglit-replayer"
+    MESA_IMAGE_TAG: &x86_test-gl "2020-12-22-runner"

 # Debian 10 based x86 test image for VK
 x86_test-vk:
  extends: .use-x86_test-base
  variables:
-    MESA_IMAGE_TAG: &x86_test-vk "2020-12-19-build-piglit"
+    MESA_IMAGE_TAG: &x86_test-vk "2020-12-22-runner"

 # Debian 10 based ARM build image
 arm_build:
@ -408,7 +408,7 @@ arm64_test:
  extends:
    - .use-arm_test-base
  variables:
-    MESA_IMAGE_TAG: &arm64_test "2020-12-19-build-piglit"
+    MESA_IMAGE_TAG: &arm64_test "2020-12-22-runner"

 .use-arm64_test:
  variables:
@ -422,7 +422,7 @@ armhf_test:
  extends:
    - .use-arm_test-base
  variables:
-    MESA_IMAGE_TAG: &armhf_test "2020-12-19-build-piglit"
+    MESA_IMAGE_TAG: &armhf_test "2020-12-22-runner"

 .use-armhf_test:
  variables:
@ -1179,6 +1179,9 @@ radv-raven-traces:
 .deqp-test:
  script:
    - ./install/deqp-runner.sh
+  artifacts:
+    reports:
+      junit: results/junit.xml

 .deqp-test-vk:
  extends:
@ -1412,6 +1415,8 @@ arm64_a630_vk_sysmem:
      - serial*.txt
    exclude:
      - results/*.shader_cache
+    reports:
+      junit: results/junit.xml

 .freedreno-test:
  extends:
--- a/.gitlab-ci/build-deqp-runner.sh
+++ b/.gitlab-ci/build-deqp-runner.sh
@ -4,6 +4,6 @@ set -ex

 cargo install deqp-runner \
  -j ${FDO_CI_CONCURRENT:-4} \
-  --version 0.1.5 \
+  --version 0.4.0 \
  --root /usr/local \
  $EXTRA_CARGO_ARGS
--- a/.gitlab-ci/deqp-freedreno-a530-fails.txt
+++ b/.gitlab-ci/deqp-freedreno-a530-fails.txt
@ -39,6 +39,7 @@ dEQP-GLES3.functional.transform_feedback.random.interleaved.lines.3,Fail
 dEQP-GLES3.functional.transform_feedback.random.separate.points.3,Fail
 dEQP-GLES3.functional.transform_feedback.random_full_array_capture.separate.triangles.3,Fail
 dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.r32i_rgba8,Fail
+dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba32f_rgba32ui,Fail
 dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba8_snorm_r32ui,Fail
 dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba8i_r32f,Fail
 dEQP-GLES31.functional.image_load_store.cube.load_store.r32f_single_layer,Fail
--- a/.gitlab-ci/deqp-freedreno-a630-fails.txt
+++ b/.gitlab-ci/deqp-freedreno-a630-fails.txt
@ -1,8 +1,14 @@
 KHR-GL30.transform_feedback.api_errors_test,Fail
+KHR-GL30.transform_feedback.capture_vertex_interleaved_test,Fail
+KHR-GL30.transform_feedback.capture_vertex_separate_test,Fail
+KHR-GL30.transform_feedback.discard_vertex_test,Fail
 KHR-GL30.transform_feedback.draw_xfb_feedbackk_test,Crash
 KHR-GL30.transform_feedback.draw_xfb_instanced_test,Crash
 KHR-GL30.transform_feedback.draw_xfb_stream_instanced_test,Crash
 KHR-GL30.transform_feedback.draw_xfb_test,Crash
+KHR-GL30.transform_feedback.get_xfb_varying,Fail
+KHR-GL30.transform_feedback.query_vertex_interleaved_test,Fail
+KHR-GL30.transform_feedback.query_vertex_separate_test,Fail
 dEQP-VK.api.object_management.single_alloc_callbacks.compute_pipeline,Fail
 dEQP-VK.compute.indirect_dispatch.gen_in_compute.multiple_groups_multiple_invocations,Fail
 dEQP-VK.compute.indirect_dispatch.upload_buffer.multiple_groups,Fail
@ -29,6 +35,7 @@ dEQP-VK.multiview.renderpass2.masks.max_multi_view_view_count,Fail
 dEQP-VK.pipeline.extended_dynamic_state.after_pipelines.depth_compare_greater_equal_greater,Fail
 dEQP-VK.pipeline.extended_dynamic_state.before_draw.depth_compare_always_greater,Fail
 dEQP-VK.pipeline.multisample.alpha_to_coverage_unused_attachment.samples_4.alpha_invisible,Fail
+dEQP-VK.pipeline.push_descriptor.compute.binding3_numcalls2_sampler,Crash
 dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_16_64_6.samples_2.d24_unorm_s8_uint.depth_zero,Fail
 dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_16_64_6.samples_4.x8_d24_unorm_pack32.depth_zero,Fail
 dEQP-VK.renderpass2.suballocation.attachment_allocation.input_output.7,Fail
@ -38,11 +45,15 @@ dEQP-VK.spirv_assembly.instruction.graphics.opquantize.negative_too_small_tesse,
 dEQP-VK.spirv_assembly.instruction.graphics.opquantize.round_to_inf_tesse,Fail
 dEQP-VK.spirv_assembly.instruction.graphics.opquantize.spec_const_carry_to_exponent_tesse,Fail
 dEQP-VK.spirv_assembly.instruction.graphics.opquantize.spec_const_negative_round_up_or_round_down_tesse,Fail
+dEQP-VK.tessellation.invariance.inner_triangle_set.triangles_equal_spacing,Fail
 dEQP-VK.tessellation.invariance.outer_edge_division.triangles_fractional_even_spacing,Fail
 dEQP-VK.tessellation.invariance.outer_edge_index_independence.triangles_equal_spacing_ccw,Fail
+dEQP-VK.tessellation.invariance.outer_edge_index_independence.triangles_fractional_even_spacing_cw,Fail
+dEQP-VK.tessellation.invariance.outer_edge_index_independence.quads_fractional_even_spacing_ccw,Fail
 dEQP-VK.tessellation.invariance.outer_edge_symmetry.isolines_equal_spacing_cw,Fail
 dEQP-VK.tessellation.invariance.outer_edge_symmetry.quads_fractional_odd_spacing_ccw,Fail
 dEQP-VK.tessellation.invariance.outer_edge_symmetry.triangles_fractional_odd_spacing_cw,Fail
+dEQP-VK.tessellation.invariance.outer_triangle_set.quads_fractional_odd_spacing,Fail
 dEQP-VK.tessellation.invariance.primitive_set.isolines_fractional_odd_spacing_ccw,Fail
 dEQP-VK.tessellation.invariance.primitive_set.quads_fractional_odd_spacing_cw,Fail
 dEQP-VK.tessellation.invariance.primitive_set.triangles_fractional_even_spacing_ccw,Fail
--- a/.gitlab-ci/deqp-freedreno-a630-skips.txt
+++ b/.gitlab-ci/deqp-freedreno-a630-skips.txt
@ -18,6 +18,7 @@ dEQP-VK.spirv_assembly.instruction.graphics.spirv_ids_abuse.lots_ids_tesse
 dEQP-VK.tessellation.invariance.outer_edge_division.quads_fractional_odd_spacing

 # Timeout (VK-GL-CTS 1.2.5.0)
+dEQP-VK.geometry.layered.cube_array.36_36_12.readback
 dEQP-VK.geometry.layered.cube_array.64_64_12.readback

 # Crashes likely caused by https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2701
--- a/.gitlab-ci/deqp-lvp-fails.txt
+++ b/.gitlab-ci/deqp-lvp-fails.txt
@ -4,6 +4,7 @@ dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8_unorm.r16g16
 dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.a2b10g10r10_unorm_pack32.optimal_optimal_linear,Fail
 dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.r16g16_unorm.general_optimal_linear,Fail
 dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.r16g16b16a16_unorm.linear_general_linear,Fail
+dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8_unorm.r16g16b16a16_unorm.general_optimal_linear,Fail
 dEQP-VK.glsl.builtin.precision.pow.highp.vec3,Fail
 dEQP-VK.glsl.texture_functions.query.texturequerylod.sampler1d_fixed_fragment,Fail
 dEQP-VK.glsl.texture_functions.query.texturequerylod.sampler2darray_fixed_fragment,Fail
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@ -120,6 +120,7 @@ run_cts() {
        --deqp $deqp \
        --output $RESULTS \
        --caselist $caselist \
+        --testlog-to-xml  /deqp/executor/testlog-to-xml \
        $JOB \
        $SUMMARY_LIMIT \
 	$DEQP_RUNNER_OPTIONS \
@ -170,31 +171,6 @@ report_flakes() {

 }

-# Generate junit results
-generate_junit() {
-    results=$1
-    echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
-    echo "<testsuites>"
-    echo "<testsuite name=\"$DEQP_VER-$CI_NODE_INDEX\">"
-    while read line; do
-        testcase=${line%,*}
-        result=${line#*,}
-        # avoid counting Skip's in the # of tests:
-        if [ "$result" = "Skip" ]; then
-            continue;
-        fi
-        echo "<testcase name=\"$testcase\">"
-        if [ "$result" != "Pass" ]; then
-            echo "<failure type=\"$result\">"
-            echo "$result: See $CI_JOB_URL/artifacts/results/$testcase.xml"
-            echo "</failure>"
-        fi
-        echo "</testcase>"
-    done < $results
-    echo "</testsuite>"
-    echo "</testsuites>"
-}
-
 parse_renderer() {
    RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
    VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
@ -266,25 +242,24 @@ DEQP_EXITCODE=$?
 echo "System load: $(cut -d' ' -f1-3 < /proc/loadavg)"
 echo "# of CPU cores: $(cat /proc/cpuinfo | grep processor | wc -l)"

-# junit is disabled, because it overloads gitlab.freedesktop.org to parse it.
-# quiet generate_junit $RESULTS_CSV > $RESULTS/results.xml
+# Remove all but the first 50 individual XML files uploaded as artifacts, to
+# save fd.o space when you break everything.
+find $RESULTS -name \*.xml | \
+    sort -n |
+    sed -n '1,+49!p' | \
+    xargs rm -f

-# Turn up to the first 50 individual test QPA files from failures or flakes into
-# XML results you can view from the browser.
-qpas=`find $RESULTS -name \*.qpa -a ! -name deqp-info.qpa`
-if [ -n "$qpas" ]; then
-    shard_qpas=`echo "$qpas" | grep dEQP- | head -n 50`
-    for qpa in $shard_qpas; do
-        xml=`echo $qpa | sed 's|\.qpa|.xml|'`
-        /deqp/executor/testlog-to-xml $qpa $xml
-    done
+# If any QPA XMLs are there, then include the XSL/CSS in our artifacts.
+find $RESULTS -name \*.xml \
+    -exec cp /deqp/testlog.css /deqp/testlog.xsl "$RESULTS/" ";" \
+    -quit

-    cp /deqp/testlog.css "$RESULTS/"
-    cp /deqp/testlog.xsl "$RESULTS/"
-
-    # Remove all the QPA files (extracted or not) now that we have the XML we want.
-    echo $qpas | xargs rm -f
-fi
+deqp-runner junit \
+   --testsuite $DEQP_VER \
+   --results $RESULTS/failures.csv \
+   --output $RESULTS/junit.xml \
+   --limit 50 \
+   --template "See https://$CI_PROJECT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml"

 # Report the flakes to the IRC channel for monitoring (if configured):
 quiet report_flakes $RESULTS_CSV
--- a/.gitlab-ci/deqp-virgl-gl-fails.txt
+++ b/.gitlab-ci/deqp-virgl-gl-fails.txt
@ -4678,7 +4678,7 @@ KHR-GL30.transform_feedback.query_vertex_separate_test,Fail
 KHR-GL31.transform_feedback.capture_vertex_interleaved_test,Fail
 KHR-GL31.transform_feedback.capture_vertex_separate_test,Fail
 KHR-GL31.transform_feedback.discard_vertex_test,Fail
-KHR-GL31.transform_feedback.draw_xfb_instanced_test,Fail
+KHR-GL31.transform_feedback.draw_xfb_instanced_test,Crash
 KHR-GL31.transform_feedback.draw_xfb_stream_instanced_test,Crash
 KHR-GL31.transform_feedback.query_vertex_interleaved_test,Fail
 KHR-GL31.transform_feedback.query_vertex_separate_test,Fail
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@ -1,5 +1,5 @@
 variables:
-  DISTRIBUTION_TAG: "2020-12-19-build-piglit"
+  DISTRIBUTION_TAG: "2020-12-22-runner"

 .kernel+rootfs:
  stage: container-2