ci: Include trace replay support in ARM rootfses.

Builds the renderdoc and apitrace programs so we can replay GL traces on DUTs. [Separated out from 5472's commit that also enabled the jobs in LAVA, dropped unnecessary python packages from arm_build, fixed up arm64_test build, traces-db in baremetal, new commit message by anholt] Signed-off-by: Rohan Garg <rohan.garg@collabora.com> Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com> Reviewed-by: Eric Anholt <eric@anholt.net> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5433>
2020-01-28 15:19:53 +01:00 · 2020-01-28 15:19:53 +01:00 · 7406d627c8
parent acf9d8b75d
commit 7406d627c8
9 changed files with 105 additions and 13 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -272,7 +272,7 @@ arm_build:
    - .fdo.container-build@debian@arm64v8
    - .container
  variables:
-    FDO_DISTRIBUTION_TAG: &arm_build "2020-06-22-stable"
+    FDO_DISTRIBUTION_TAG: &arm_build "2020-06-22-tracie"

 .use-arm_build:
  variables:
@ -287,7 +287,7 @@ arm_test-base:
    - .fdo.container-build@debian
    - .container
  variables:
-    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-06-22-stable"
+    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-06-22-tracie"

 .use-arm_test-base:
  extends:
@ -305,7 +305,7 @@ arm64_test:
  extends:
    - .use-arm_test-base
  variables:
-    FDO_DISTRIBUTION_TAG: &arm64_test "2020-06-22-ephemeral"
+    FDO_DISTRIBUTION_TAG: &arm64_test "2020-06-22-tracie"

 .use-arm64_test:
  variables:
--- a/.gitlab-ci/bare-metal/fastboot.sh
+++ b/.gitlab-ci/bare-metal/fastboot.sh
@ -60,7 +60,8 @@ rsync -a --delete $BM_ROOTFS/ rootfs/
 # these devices use it and it would take up space in the initrd.
 pushd rootfs
 find -H | \
-  egrep -v "external/(openglcts|vulkancts|amber|glslang|spirv-tools)" | \
+  egrep -v "external/(openglcts|vulkancts|amber|glslang|spirv-tools)" |
+  egrep -v "traces-db|apitrace|renderdoc|python" | \
  cpio -H newc -o | \
  xz --check=crc32 -T4 - > $CI_PROJECT_DIR/rootfs.cpio.gz
 popd
--- a/.gitlab-ci/build-apitrace.sh
+++ b/.gitlab-ci/build-apitrace.sh
@ -9,22 +9,25 @@ WAFFLE_VERSION="e3c995d9a2693b687501715b6550619922346089"
 git clone https://gitlab.freedesktop.org/mesa/waffle.git --single-branch --no-checkout /waffle
 pushd /waffle
 git checkout "$WAFFLE_VERSION"
-cmake -B_build -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=Release .
-make -C _build -j4 install
+cmake -B_build -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=Release $EXTRA_CMAKE_ARGS .
+make -C _build install
+mkdir -p build/lib build/bin
+cp _build/lib/libwaffle-1.so build/lib/libwaffle-1.so.0
+cp _build/bin/wflinfo build/bin/wflinfo
+${STRIP_CMD:-strip} build/lib/* build/bin/*
+find . -not -path './build' -not -path './build/*' -delete
 popd
-rm -rf /waffle

 APITRACE_VERSION="9.0"

 git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout /apitrace
 pushd /apitrace
 git checkout "$APITRACE_VERSION"
-cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/
+cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/ $EXTRA_CMAKE_ARGS
 ninja -C _build
 mkdir build
 cp _build/apitrace build
-cp _build/glretrace build
 cp _build/eglretrace build
-strip build/*
+${STRIP_CMD:-strip} build/*
 find . -not -path './build' -not -path './build/*' -delete
 popd
--- a/.gitlab-ci/build-renderdoc.sh
+++ b/.gitlab-ci/build-renderdoc.sh
@ -7,11 +7,11 @@ RENDERDOC_VERSION=da02e88201dc3b64316fc33ce6ff69cc729689aa
 git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout /renderdoc
 pushd /renderdoc
 git checkout "$RENDERDOC_VERSION"
-cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release
+cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release $EXTRA_CMAKE_ARGS
 ninja -C _build
 mkdir -p build/lib
+${STRIP_CMD:-strip} _build/lib/*.so
 cp _build/lib/renderdoc.so build/lib
 cp _build/lib/librenderdoc.so build/lib
-strip build/lib/*
 find . -not -path './build' -not -path './build/*' -delete
 popd
--- a/.gitlab-ci/container/arm_build.sh
+++ b/.gitlab-ci/container/arm_build.sh
@ -12,6 +12,8 @@ apt-get update
 apt-get -y install \
 	abootimg \
 	android-sdk-ext4-utils \
+	autoconf \
+	automake \
 	bc \
 	bison \
 	ccache \
@ -24,28 +26,45 @@ apt-get -y install \
 	g++ \
 	git \
 	lavacli \
+	libboost-dev:armhf \
+	libboost-dev \
 	libdrm-dev:armhf \
+	libdrm-dev \
 	libegl1-mesa-dev \
 	libegl1-mesa-dev:armhf \
 	libelf-dev \
 	libelf-dev:armhf \
 	libexpat1-dev \
 	libexpat1-dev:armhf \
+	libgbm-dev \
+	libgbm-dev:armhf \
 	libgles2-mesa-dev \
 	libgles2-mesa-dev:armhf \
+	libpcre3-dev \
+	libpcre3-dev:armhf \
 	libpng-dev \
 	libpng-dev:armhf \
+	libpython3-dev \
+	libpython3-dev:armhf \
 	libssl-dev \
 	libvulkan-dev \
+	libvulkan-dev \
 	libvulkan-dev:armhf \
+	libxcb-keysyms1-dev \
+	libxcb-keysyms1-dev:armhf \
 	llvm-7-dev:armhf \
 	llvm-8-dev \
 	pkg-config \
 	python \
+	python3-dev \
 	python3-distutils \
 	python3-setuptools \
 	python3-mako \
 	python3-serial \
+	qt5-default \
+	qt5-qmake \
+	qtbase5-dev \
+	qtbase5-dev:armhf \
 	unzip \
 	wget \
 	xz-utils \
--- a/.gitlab-ci/container/baremetal_build.sh
+++ b/.gitlab-ci/container/baremetal_build.sh
@ -3,24 +3,38 @@
 set -e
 set -o xtrace

+ROOTFS=/lava-files/rootfs-${arch}
+
 dpkg --add-architecture $arch
 apt-get update

 # Cross-build test deps
 BAREMETAL_EPHEMERAL=" \
+        autoconf \
+        automake \
        crossbuild-essential-$arch \
+        git-lfs \
        libdrm-dev:$arch \
+        libboost-dev:$arch \
        libegl1-mesa-dev:$arch \
        libelf-dev:$arch \
        libexpat1-dev:$arch \
        libffi-dev:$arch \
        libgbm-dev:$arch \
        libgles2-mesa-dev:$arch \
+        libpcre3-dev:$arch \
        libpng-dev:$arch \
+        libpython3-dev:$arch \
        libstdc++6:$arch \
        libtinfo-dev:$arch \
        libegl1-mesa-dev:$arch \
-        libvulkan-dev:$arch
+        libvulkan-dev:$arch \
+        libxcb-keysyms1-dev:$arch \
+        libpython3-dev:$arch \
+        python3-dev \
+        qt5-default \
+        qt5-qmake \
+        qtbase5-dev:$arch \
        "

 apt-get install -y --no-remove $BAREMETAL_EPHEMERAL
@ -37,6 +51,20 @@ mkdir /var/cache/apt/archives/$arch

 DEBIAN_ARCH=$arch INCLUDE_VK_CTS=1 . .gitlab-ci/container/lava_arm.sh

+############### Store traces
+# Clone the traces-db at container build time so we don't have to pull traces
+# per run (too much egress cost for fd.o).
+git clone \
+    --depth 1 \
+    -b mesa-ci-2020-06-08 \
+    https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db.git \
+    $ROOTFS/traces-db
+rm -rf $ROOTFS/traces-db/.git
+find $ROOTFS/traces-db -type f \
+     -a -not -name '*.trace' \
+     -a -not -name '*.rdc' \
+     -delete
+
 ccache --show-stats

 . .gitlab-ci/container/container_post_build.sh
--- a/.gitlab-ci/container/lava_arm.sh
+++ b/.gitlab-ci/container/lava_arm.sh
@ -43,6 +43,25 @@ fi
 mv /deqp /lava-files/rootfs-${DEBIAN_ARCH}/.


+############### Build apitrace
+. .gitlab-ci/build-apitrace.sh
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
+mv /apitrace/build /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
+rm -rf /apitrace
+
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/waffle
+mv /waffle/build /lava-files/rootfs-${DEBIAN_ARCH}/waffle
+rm -rf /waffle
+
+
+############### Build renderdoc
+EXTRA_CMAKE_ARGS+=" -DENABLE_XCB=false"
+. .gitlab-ci/build-renderdoc.sh
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
+mv /renderdoc/build /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
+rm -rf /renderdoc
+
+
 ############### Cross-build kernel
 KERNEL_URL="https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.5-panfrost-fixes/linux-v5.5-panfrost-fixes.tar.gz"

--- a/.gitlab-ci/create-rootfs.sh
+++ b/.gitlab-ci/create-rootfs.sh
@ -11,8 +11,16 @@ apt-get -y install --no-install-recommends \
    libexpat1 \
    libdrm2 \
    libdrm-nouveau2 \
+    libx11-6 \
+    libx11-xcb1 \
    firmware-qcom-media \
    netcat-openbsd \
+    python3 \
+    libpython3.7 \
+    python3-pil \
+    python3-pytest \
+    python3-requests \
+    python3-yaml \
    wget \
    xz-utils

@ -72,6 +80,7 @@ rm -rf /var/log/*
 # Dropping documentation, localization, i18n files, etc
 rm -rf /usr/share/doc/*
 rm -rf /usr/share/locale/*
+rm -rf /usr/share/X11/locale/*
 rm -rf /usr/share/man
 rm -rf /usr/share/i18n/*
 rm -rf /usr/share/info/*
@ -115,10 +124,20 @@ UNNEEDED_PACKAGES="apt libapt-pkg6.0 "\
 "passwd "\
 "libsemanage1 libsemanage-common "\
 "libsepol1 "\
+"gzip "\
 "gpgv "\
 "hostname "\
 "adduser "\
 "debian-archive-keyring "\
+"libegl1-mesa-dev "\
+"libegl-mesa0 "\
+"libgl1-mesa-dev "\
+"libgl1-mesa-dri "\
+"libglapi-mesa "\
+"libgles2-mesa-dev "\
+"libglx-mesa0 "\
+"mesa-common-dev "\
+"libz3-4 "\

 # Removing unneeded packages
 for PACKAGE in ${UNNEEDED_PACKAGES}
--- a/.gitlab-ci/tracie-runner-gl.sh
+++ b/.gitlab-ci/tracie-runner-gl.sh
@ -17,6 +17,9 @@ export LD_LIBRARY_PATH="/waffle/build/lib:$LD_LIBRARY_PATH"
 # Set environment for apitrace executable.
 export PATH="/apitrace/build:$PATH"

+# Set environment for wflinfo executable.
+export PATH="/waffle/build/bin:$PATH"
+
 # Use the surfaceless EGL platform.
 export EGL_PLATFORM="surfaceless"
 export DISPLAY=