panfrost: ci: Add support for running the tests on RK3288

Build artifacts for armhf and schedule them on a Veyron Chromebook with
RK3288.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>

src/gallium/drivers/panfrost/ci/Dockerfile

@@ -8,12 +8,15 @@ RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d
 RUN echo 'exit 101' >> /usr/sbin/policy-rc.d
 RUN chmod +x /usr/sbin/policy-rc.d
 
-RUN dpkg --add-architecture arm64
+############### Install packages for building
+
+ARG DEBIAN_ARCH
+RUN dpkg --add-architecture ${DEBIAN_ARCH}
 RUN echo deb-src http://deb.debian.org/debian testing main >> /etc/apt/sources.list
 RUN apt-get update && \
     apt-get -y install ca-certificates && \
     apt-get -y install --no-install-recommends \
-      crossbuild-essential-arm64 \
+      crossbuild-essential-${DEBIAN_ARCH} \
       meson \
       g++ \
       git \
@@ -45,46 +48,48 @@ RUN apt-get update && \
       qemu-user-static \
       cpio \
       \
-      libdrm-dev:arm64 \
+      libdrm-dev:${DEBIAN_ARCH} \
-      libx11-dev:arm64 \
+      libx11-dev:${DEBIAN_ARCH} \
-      libxxf86vm-dev:arm64 \
+      libxxf86vm-dev:${DEBIAN_ARCH} \
-      libexpat1-dev:arm64 \
+      libexpat1-dev:${DEBIAN_ARCH} \
-      libsensors-dev:arm64 \
+      libsensors-dev:${DEBIAN_ARCH} \
-      libxfixes-dev:arm64 \
+      libxfixes-dev:${DEBIAN_ARCH} \
-      libxdamage-dev:arm64 \
+      libxdamage-dev:${DEBIAN_ARCH} \
-      libxext-dev:arm64 \
+      libxext-dev:${DEBIAN_ARCH} \
-      x11proto-dev:arm64 \
+      x11proto-dev:${DEBIAN_ARCH} \
-      libx11-xcb-dev:arm64 \
+      libx11-xcb-dev:${DEBIAN_ARCH} \
-      libxcb-dri2-0-dev:arm64 \
+      libxcb-dri2-0-dev:${DEBIAN_ARCH} \
-      libxcb-glx0-dev:arm64 \
+      libxcb-glx0-dev:${DEBIAN_ARCH} \
-      libxcb-xfixes0-dev:arm64 \
+      libxcb-xfixes0-dev:${DEBIAN_ARCH} \
-      libxcb-dri3-dev:arm64 \
+      libxcb-dri3-dev:${DEBIAN_ARCH} \
-      libxcb-present-dev:arm64 \
+      libxcb-present-dev:${DEBIAN_ARCH} \
-      libxcb-randr0-dev:arm64 \
+      libxcb-randr0-dev:${DEBIAN_ARCH} \
-      libxcb-sync-dev:arm64 \
+      libxcb-sync-dev:${DEBIAN_ARCH} \
-      libxrandr-dev:arm64 \
+      libxrandr-dev:${DEBIAN_ARCH} \
-      libxshmfence-dev:arm64 \
+      libxshmfence-dev:${DEBIAN_ARCH} \
-      libelf-dev:arm64 \
+      libelf-dev:${DEBIAN_ARCH} \
-      libwayland-dev:arm64 \
+      libwayland-dev:${DEBIAN_ARCH} \
-      libwayland-egl-backend-dev:arm64 \
+      libwayland-egl-backend-dev:${DEBIAN_ARCH} \
-      libclang-7-dev:arm64 \
+      libclang-7-dev:${DEBIAN_ARCH} \
-      zlib1g-dev:arm64 \
+      zlib1g-dev:${DEBIAN_ARCH} \
-      libglvnd-core-dev:arm64 \
+      libglvnd-core-dev:${DEBIAN_ARCH} \
-      wayland-protocols:arm64 \
+      wayland-protocols:${DEBIAN_ARCH} \
-      libpng-dev:arm64 && \
+      libpng-dev:${DEBIAN_ARCH} && \
     rm -rf /var/lib/apt/lists
 
+############### Cross-build dEQP
+ARG GCC_ARCH
 RUN mkdir -p /artifacts/rootfs/deqp                                             && \
   wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip && \
-  unzip opengl-es-cts-3.2.5.0.zip -d /                                          && \
+  unzip -q opengl-es-cts-3.2.5.0.zip -d /                                       && \
   rm opengl-es-cts-3.2.5.0.zip                                                  && \
   cd /VK-GL-CTS-opengl-es-cts-3.2.5.0                                           && \
   python3 external/fetch_sources.py                                             && \
   cd /artifacts/rootfs/deqp                                                     && \
   cmake -DDEQP_TARGET=wayland                                                      \
     -DCMAKE_BUILD_TYPE=Release                                                     \
-    -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc                                       \
+    -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc                                             \
-    -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++                                     \
+    -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++                                           \
     /VK-GL-CTS-opengl-es-cts-3.2.5.0                                            && \
   make -j$(nproc)                                                               && \
   rm -rf /artifacts/rootfs/deqp/external                                        && \
@@ -100,22 +105,32 @@ RUN mkdir -p /artifacts/rootfs/deqp
   du -sh *                                                                      && \
   rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
 
+
+############### Cross-build kernel
+
+ARG KERNEL_ARCH
+ARG DEFCONFIG
+ARG DEVICE_TREES
+ARG KERNEL_IMAGE_NAME
 # TODO: Switch to 5.2-rc* when the time comes
-COPY arm64.config /panfrost-ci/
+COPY ${KERNEL_ARCH}.config /panfrost-ci/
 RUN mkdir -p /kernel                                                                   && \
   wget https://github.com/freedesktop/drm-misc/archive/drm-misc-next-2019-04-18.tar.gz && \
-  tar xvfz drm-misc-next-2019-04-18.tar.gz -C /kernel --strip-components=1             && \
+  tar xfz drm-misc-next-2019-04-18.tar.gz -C /kernel --strip-components=1              && \
   rm drm-misc-next-2019-04-18.tar.gz                                                   && \
   cd /kernel                                                                           && \
-  ARCH=arm64 CROSS_COMPILE="aarch64-linux-gnu-" ./scripts/kconfig/merge_config.sh arch/arm64/configs/defconfig /panfrost-ci/arm64.config && \
+  ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" ./scripts/kconfig/merge_config.sh ${DEFCONFIG} /panfrost-ci/${KERNEL_ARCH}.config && \
-  ARCH=arm64 CROSS_COMPILE="aarch64-linux-gnu-" make -j12 Image dtbs                   && \
+  ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" make -j12 ${KERNEL_IMAGE_NAME} dtbs && \
-  cp arch/arm64/boot/Image /artifacts/.                                                && \
+  cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/.                        && \
-  cp arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dtb /artifacts/.                    && \
+  cp ${DEVICE_TREES} /artifacts/.                                                      && \
   rm -rf /kernel
 
+
+############### Create rootfs
+
 COPY create-rootfs.sh /artifacts/rootfs/
-RUN debootstrap --variant=minbase --arch=arm64 testing /artifacts/rootfs/ http://deb.debian.org/debian && \
+RUN debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian && \
-    chroot /artifacts/rootfs sh /create-rootfs.sh                                                      && \
+    chroot /artifacts/rootfs sh /create-rootfs.sh                                                               && \
     rm /artifacts/rootfs/create-rootfs.sh
 
 ENTRYPOINT [""]

src/gallium/drivers/panfrost/ci/arm.config

@@ -0,0 +1,45 @@
+CONFIG_LOCALVERSION="ccu"
+
+CONFIG_DEBUG_KERNEL=y
+
+CONFIG_DEVFREQ_GOV_PERFORMANCE=y
+CONFIG_DEVFREQ_GOV_POWERSAVE=y
+CONFIG_DEVFREQ_GOV_USERSPACE=y
+CONFIG_DEVFREQ_GOV_PASSIVE=y
+CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
+
+CONFIG_DRM=y
+CONFIG_DRM_ROCKCHIP=y
+CONFIG_DRM_PANFROST=y
+CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_PWM_CROS_EC=y
+CONFIG_BACKLIGHT_PWM=y
+
+CONFIG_ROCKCHIP_CDN_DP=n
+
+CONFIG_SPI_ROCKCHIP=y
+CONFIG_PWM_ROCKCHIP=y
+CONFIG_PHY_ROCKCHIP_DP=y
+CONFIG_DWMAC_ROCKCHIP=y
+
+CONFIG_MFD_RK808=y
+CONFIG_REGULATOR_RK808=y
+CONFIG_RTC_DRV_RK808=y
+CONFIG_COMMON_CLK_RK808=y
+
+CONFIG_REGULATOR_FAN53555=y
+CONFIG_REGULATOR=y
+
+CONFIG_REGULATOR_VCTRL=y
+
+CONFIG_KASAN=n
+CONFIG_KASAN_INLINE=n
+CONFIG_STACKTRACE=n
+
+CONFIG_TMPFS=y
+
+CONFIG_PROVE_LOCKING=n
+CONFIG_DEBUG_LOCKDEP=n
+CONFIG_SOFTLOCKUP_DETECTOR=n
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=n
+

src/gallium/drivers/panfrost/ci/deqp-runner.sh

@@ -6,8 +6,8 @@ set -x
 BATCH_SIZE=5000
 
 export XDG_RUNTIME_DIR=/tmp
-export LIBGL_DRIVERS_PATH=/mesa/lib/aarch64-linux-gnu/dri/
+export LIBGL_DRIVERS_PATH=/mesa/lib/dri/
-export LD_LIBRARY_PATH=/mesa/lib/aarch64-linux-gnu
+export LD_LIBRARY_PATH=/mesa/lib/
 export XDG_CONFIG_HOME=$(pwd)
 
 echo "[core]\nidle-time=0\nrequire-input=false\n[shell]\nlocking=false" > weston.ini

src/gallium/drivers/panfrost/ci/generate_lava.py

@@ -6,6 +6,9 @@ import argparse
 parser = argparse.ArgumentParser()
 parser.add_argument("--template")
 parser.add_argument("--base-artifacts-url")
+parser.add_argument("--arch")
+parser.add_argument("--device-type")
+parser.add_argument("--kernel-image-name")
 args = parser.parse_args()
 
 env = Environment(loader = FileSystemLoader('.'), trim_blocks=True, lstrip_blocks=True)
@@ -13,5 +16,8 @@ template = env.get_template(args.template)
 
 values = {}
 values['base_artifacts_url'] = args.base_artifacts_url
+values['arch'] = args.arch
+values['device_type'] = args.device_type
+values['kernel_image_name'] = args.kernel_image_name
 
 print(template.render(values))

src/gallium/drivers/panfrost/ci/gitlab-ci.yml

@@ -7,37 +7,51 @@ stages:
   - test
 
 # Build Docker image with deqp, the rootfs and the build deps for Mesa
-build-container:
+.build-container:
   stage: build-container
   when: always
   services:
     - docker:dind
   image: docker:latest
-  before_script:
-    - mkdir -p results
   script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
-    - time docker pull $CI_REGISTRY_IMAGE:latest || true
+    - time docker pull $CI_REGISTRY_IMAGE/${DEBIAN_ARCH}:latest || true
     - time docker build
-             -t $CI_REGISTRY_IMAGE
+             -t $CI_REGISTRY_IMAGE/${DEBIAN_ARCH}
-             --cache-from $CI_REGISTRY_IMAGE:latest
+             --cache-from $CI_REGISTRY_IMAGE/${DEBIAN_ARCH}:latest
+             --build-arg DEBIAN_ARCH=${DEBIAN_ARCH}
+             --build-arg GCC_ARCH=${GCC_ARCH}
+             --build-arg KERNEL_ARCH=${KERNEL_ARCH}
+             --build-arg DEFCONFIG=${DEFCONFIG}
+             --build-arg DEVICE_TREES=${DEVICE_TREES}
+             --build-arg KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME}
              src/gallium/drivers/panfrost/ci
-             2>&1 | tee results/docker_build_log.txt
+    - time docker push $CI_REGISTRY_IMAGE/${DEBIAN_ARCH}:latest
-    - time docker push $CI_REGISTRY_IMAGE:latest 2>&1 | tee results/docker_push_log.txt
+    - docker history $CI_REGISTRY_IMAGE/${DEBIAN_ARCH}:latest
 
-    - docker history $CI_REGISTRY_IMAGE:latest
+build-container-armhf:
+  extends: .build-container
+  variables:
+    DEBIAN_ARCH: "armhf"
+    GCC_ARCH: "arm-linux-gnueabihf"
+    KERNEL_ARCH: "arm"
+    DEFCONFIG: "arch/arm/configs/multi_v7_defconfig"
+    DEVICE_TREES: "arch/arm/boot/dts/rk3288-veyron-jaq.dtb"
+    KERNEL_IMAGE_NAME: "zImage"
 
-    - "grep -o 'digest: sha256:[0-9a-f]\\+' results/docker_push_log.txt |
+build-container-arm64:
-       cut -f 2 -d ' ' |
+  extends: .build-container
-       tee results/docker_image_digest.txt"
+  variables:
-  artifacts:
+    DEBIAN_ARCH: "arm64"
-    when: always
+    GCC_ARCH: "aarch64-linux-gnu"
-    paths:
+    KERNEL_ARCH: "arm64"
-      - results/
+    DEFCONFIG: "arch/arm64/configs/defconfig"
+    DEVICE_TREES: "arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dtb"
+    KERNEL_IMAGE_NAME: "Image"
 
-build:
+.build:
   stage: build
-  image: $CI_REGISTRY_IMAGE:latest
+  image: $CI_REGISTRY_IMAGE/${DEBIAN_ARCH}:latest
   cache:
     paths:
       - ccache
@@ -50,10 +64,11 @@ build:
     - export PATH="/usr/lib/ccache:$PATH"
     - ccache -s
 
-    - /usr/share/meson/debcrossgen --arch arm64 -o /tmp/cross_file.txt
+    # Build Mesa
+    - /usr/share/meson/debcrossgen --arch ${DEBIAN_ARCH} -o /tmp/cross_file.txt
     - meson . mesa-build
         --cross-file /tmp/cross_file.txt
-        --libdir /artifacts/rootfs/mesa/lib/aarch64-linux-gnu
+        --libdir /artifacts/rootfs/mesa/lib/
         --buildtype release
         -Dgallium-drivers=kmsro,panfrost
         -Ddri-drivers=
@@ -61,24 +76,50 @@ build:
     - ninja -C mesa-build install
     - du -sh /artifacts/rootfs/mesa/*
     - rm -rf /artifacts/rootfs/mesa/include
+
+    # Pack rootfs
     - cp src/gallium/drivers/panfrost/ci/deqp-runner.sh /artifacts/rootfs/deqp/.
     - du -sh /artifacts/rootfs/deqp/*
     - find /artifacts/rootfs/ -type f -printf "%s\t%p\n" | sort -n
-    - cd /artifacts/rootfs/ ; find -H  |  cpio -H newc -v -o | gzip -c - > $CI_PROJECT_DIR/results/panfrost-rootfs.cpio.gz
+    - cd /artifacts/rootfs/ ; find -H  |  cpio -H newc -v -o | gzip -c - > $CI_PROJECT_DIR/results/panfrost-rootfs-${DEBIAN_ARCH}.cpio.gz
-    - cp /artifacts/Image /artifacts/rk3399-gru-kevin.dtb $CI_PROJECT_DIR/results/.
 
+    # Copy kernel and DT
+    - cp /artifacts/${KERNEL_IMAGE_NAME} /artifacts/*.dtb $CI_PROJECT_DIR/results/.
+
+    # Generate LAVA job
     - cd $CI_PROJECT_DIR
     - src/gallium/drivers/panfrost/ci/generate_lava.py
         --template src/gallium/drivers/panfrost/ci/lava-deqp.yml.jinja2
+        --arch ${DEBIAN_ARCH}
         --base-artifacts-url $CI_PROJECT_URL/-/jobs/$CI_JOB_ID/artifacts/raw/results
+        --device-type ${DEVICE_TYPE}
+        --kernel-image-name ${KERNEL_IMAGE_NAME}
         > results/lava-deqp.yml
   artifacts:
     when: always
     paths:
       - results/
 
-test:
+build-armhf:
+  extends: .build
+  variables:
+    DEBIAN_ARCH: "armhf"
+    GCC_ARCH: "arm-linux-gnueabihf"
+    DEVICE_TYPE: "rk3288-veyron-jaq"
+    KERNEL_IMAGE_NAME: "zImage"
+
+build-arm64:
+  extends: .build
+  variables:
+    DEBIAN_ARCH: "arm64"
+    GCC_ARCH: "aarch64-linux-gnu"
+    DEVICE_TYPE: "rk3399-gru-kevin"
+    KERNEL_IMAGE_NAME: "Image"
+
+.test:
   stage: test
+  tags:
+    - idle-jobs
   image: $CI_REGISTRY_IMAGE:latest
   script:
     - mkdir -p ~/.config/
@@ -89,14 +130,14 @@ test:
         username: jenkins-fdo
         token: $LAVA_TOKEN
       " > ~/.config/lavacli.yaml
-    - lava_job_id=`lavacli jobs submit $CI_PROJECT_DIR/results/lava-deqp.yml`
+    - lava_job_id=`lavacli jobs submit $CI_PROJECT_DIR/results/lava-deqp.yml` || echo $lava_job_id
-    - echo $lava_job_id
     - lavacli jobs logs $lava_job_id | grep -a -v "{'case':" | tee results/lava-deqp-$lava_job_id.log
     - lavacli jobs show $lava_job_id
     - curl "https://lava.collabora.co.uk/results/$lava_job_id/csv?user=jenkins-fdo&token=$LAVA_TOKEN" > raw_results.csv
     - cat raw_results.csv | csvcut -c 12,3 | grep dEQP-GLES2 | sort > results/results-$lava_job_id.csv
 
     # FIXME: Remove flip-flops from comparison files
+    - sed -i '/dEQP-GLES2.functional.shaders.random.all_features.fragment.12/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
     - sed -i '/dEQP-GLES2.functional.shaders.invariance.highp.loop_4/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
     - sed -i '/dEQP-GLES2.functional.texture.units.2_units.mixed.1/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
     - sed -i '/dEQP-GLES2.functional.texture.units.4_units.mixed.5/d' src/gallium/drivers/panfrost/ci/expected-failures.txt results/results-$lava_job_id.csv
@@ -115,3 +156,14 @@ test:
     when: always
     paths:
       - results/
+
+test-armhf:
+  extends: .test
+  dependencies:
+    - build-armhf
+
+test-arm64:
+  extends: .test
+  dependencies:
+    - build-arm64
+

src/gallium/drivers/panfrost/ci/lava-deqp.yml.jinja2

@@ -1,5 +1,5 @@
 job_name: panfrost-deqp
-device_type: rk3399-gru-kevin
+device_type: {{ device_type }}
 timeouts:
   job:
     minutes: 40
@@ -16,12 +16,12 @@ actions:
       minutes: 2
     to: tftp
     kernel:
-      url: {{ base_artifacts_url }}/Image
+      url: {{ base_artifacts_url }}/{{ kernel_image_name }}
     ramdisk:
-      url: {{ base_artifacts_url }}/panfrost-rootfs.cpio.gz
+      url: {{ base_artifacts_url }}/panfrost-rootfs-{{ arch }}.cpio.gz
       compression: gz
     dtb:
-      url: {{ base_artifacts_url }}/rk3399-gru-kevin.dtb
+      url: {{ base_artifacts_url }}/{{ device_type }}.dtb
     os: oe
 - boot:
     timeout:
@@ -51,7 +51,7 @@ actions:
           - mkdir -p /dev/pts
           - mount -t devpts devpts /dev/pts
           - echo 1 > /proc/sys/kernel/printk
-          - echo performance > /sys/devices/platform/ff9a0000.gpu/devfreq/devfreq0/governor
+#          - echo performance > /sys/devices/platform/ff9a0000.gpu/devfreq/devfreq0/governor
           - sh /deqp/deqp-runner.sh
         parse:
           pattern: 'Test case ''(?P<test_case_id>\S*)''..\s+(?P<result>(Pass|NotSupported|QualityWarning|CompatibilityWarning|Fail|ResourceError|Crash|Timeout|InternalError))'