gitlab-ci: More stable URL for kernel and ramdisk artifacts, for LAVA

Place the kernel and ramdisk into a place in the file server so the URL
will only change when the contents also change.

Also put the Mesa build into a separate tarball so the ramdisk's
contents don't change every build.

With proper caching in place, all devices in the same farm need only to
download the mesa tarball once, saving time.

As we switch to MinIO for making kernels and rootfs available to LAVA
devices, we can stop using Docker to distribute them.

Instead, build when needed in separate jobs that push directly to MinIO,
from where LAVA devices can download them.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Acked-by: Daniel Stone <daniels@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5515>
This commit is contained in:
Tomeu Vizoso 2020-04-01 13:07:46 +02:00 committed by Marge Bot
parent bf3d4b1add
commit dcd171f5e9
9 changed files with 140 additions and 138 deletions

View File

@ -313,7 +313,7 @@ arm_build:
- .fdo.container-build@debian@arm64v8
- .container
variables:
FDO_DISTRIBUTION_TAG: &arm_build "2020-06-22-tracie"
FDO_DISTRIBUTION_TAG: &arm_build "2020-07-07"
.use-arm_build:
variables:
@ -602,6 +602,8 @@ meson-armhf:
LLVM_VERSION: "7"
EXTRA_OPTION: >
-D llvm=disabled
UPLOAD_FOR_LAVA: 1
DEBIAN_ARCH: armhf
script:
- .gitlab-ci/meson-build.sh
- .gitlab-ci/prepare-artifacts.sh
@ -614,6 +616,8 @@ meson-arm64:
VULKAN_DRIVERS: "freedreno"
EXTRA_OPTION: >
-D llvm=disabled
UPLOAD_FOR_LAVA: 1
DEBIAN_ARCH: arm64
script:
- .gitlab-ci/meson-build.sh
- .gitlab-ci/prepare-artifacts.sh

View File

@ -3,11 +3,9 @@
set -e
set -o xtrace
############### Install packages for building
apt-get -y install ca-certificates
sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
echo 'deb https://deb.debian.org/debian buster-backports main' >/etc/apt/sources.list.d/backports.list
dpkg --add-architecture armhf
apt-get update
apt-get -y install \
abootimg \
@ -19,60 +17,34 @@ apt-get -y install \
ccache \
cmake \
cpio \
crossbuild-essential-armhf \
debootstrap \
fastboot \
flex \
g++ \
git \
lavacli \
libboost-dev:armhf \
libboost-dev \
libdrm-dev:armhf \
libdrm-dev \
libegl1-mesa-dev \
libegl1-mesa-dev:armhf \
libelf-dev \
libelf-dev:armhf \
libexpat1-dev \
libexpat1-dev:armhf \
libgbm-dev \
libgbm-dev:armhf \
libgles2-mesa-dev \
libgles2-mesa-dev:armhf \
libpcre3-dev \
libpcre3-dev:armhf \
libpng-dev \
libpng-dev:armhf \
libpython3-dev \
libpython3-dev:armhf \
libssl-dev \
libvulkan-dev \
libvulkan-dev \
libvulkan-dev:armhf \
libxcb-keysyms1-dev \
libxcb-keysyms1-dev:armhf \
llvm-7-dev:armhf \
llvm-8-dev \
pkg-config \
python \
python3-dev \
python3-distutils \
python3-setuptools \
python3-mako \
python3-serial \
qt5-default \
qt5-qmake \
qtbase5-dev \
qtbase5-dev:armhf \
python3-pip \
python3-setuptools \
unzip \
wget \
xz-utils \
zlib1g-dev
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@6f5af7e5574509726c79109e3c147cee95e81366
apt install -y --no-remove -t buster-backports \
meson
arch=armhf
. .gitlab-ci/container/cross_build.sh
. .gitlab-ci/container/container_pre_build.sh
# dependencies where we want a specific version
@ -83,17 +55,4 @@ tar -xvf $LIBDRM_VERSION.tar.bz2 && rm $LIBDRM_VERSION.tar.bz2
cd $LIBDRM_VERSION; meson build -D vc4=true -D freedreno=true -D etnaviv=true; ninja -C build install; cd ..
rm -rf $LIBDRM_VERSION
############### Generate cross build file for Meson
. .gitlab-ci/create-cross-file.sh armhf
############### Generate kernel, ramdisk, test suites, etc for LAVA jobs
KERNEL_URL="https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.5-panfrost-fixes/linux-v5.5-panfrost-fixes.tar.gz"
DEBIAN_ARCH=arm64 . .gitlab-ci/container/lava_arm.sh
DEBIAN_ARCH=armhf . .gitlab-ci/container/lava_arm.sh
apt-get purge -y \
wget
. .gitlab-ci/container/container_post_build.sh

View File

@ -7,8 +7,7 @@ export DEBIAN_FRONTEND=noninteractive
# Ephemeral packages (installed for this script and removed again at the end)
STABLE_EPHEMERAL=" \
libpciaccess-dev:$arch \
wget \
libpciaccess-dev:$arch
"
dpkg --add-architecture $arch
@ -23,8 +22,14 @@ apt-get install -y --no-remove \
libstdc++6:$arch \
libtinfo-dev:$arch
if [[ $arch == "armhf" ]]; then
LLVM=llvm-7-dev
else
LLVM=llvm-8-dev
fi
apt-get install -y --no-remove -t buster-backports \
llvm-8-dev:$arch
$LLVM:$arch
. .gitlab-ci/create-cross-file.sh $arch

111
.gitlab-ci/container/lava_arm.sh Normal file → Executable file
View File

@ -3,6 +3,15 @@
set -e
set -o xtrace
# If remote files are up-to-date, skip rebuilding them
MINIO_PATH=minio-packet.freedesktop.org/mesa-lava/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}
DONE_FILE=https://${MINIO_PATH}/done
if wget -q --method=HEAD ${DONE_FILE}; then
exit
fi
. .gitlab-ci/container/container_pre_build.sh
if [[ "$DEBIAN_ARCH" = "arm64" ]]; then
GCC_ARCH="aarch64-linux-gnu"
KERNEL_ARCH="arm64"
@ -15,6 +24,7 @@ else
DEFCONFIG="arch/arm/configs/multi_v7_defconfig"
DEVICE_TREES="arch/arm/boot/dts/rk3288-veyron-jaq.dtb arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dtb"
KERNEL_IMAGE_NAME="zImage"
. .gitlab-ci/create-cross-file.sh armhf
fi
# Determine if we're in a cross build.
@ -26,6 +36,46 @@ if [[ -e /cross_file-$DEBIAN_ARCH.txt ]]; then
export CROSS_COMPILE="${GCC_ARCH}-"
fi
apt-get update
apt-get install -y automake \
git \
bc \
cmake \
cpio \
wget \
debootstrap \
libboost-dev \
libegl1-mesa-dev \
libgbm-dev \
libgles2-mesa-dev \
libpcre3-dev \
libpng-dev \
libpython3-dev \
libssl-dev \
libvulkan-dev \
libxcb-keysyms1-dev \
python3-dev \
python3-distutils \
python3-serial \
qt5-default \
qt5-qmake \
qtbase5-dev
if [[ "$DEBIAN_ARCH" = "armhf" ]]; then
apt-get install -y libboost-dev:armhf \
libegl1-mesa-dev:armhf \
libelf-dev:armhf \
libgbm-dev:armhf \
libgles2-mesa-dev:armhf \
libpcre3-dev:armhf \
libpng-dev:armhf \
libpython3-dev:armhf \
libvulkan-dev:armhf \
libxcb-keysyms1-dev:armhf \
qtbase5-dev:armhf
fi
############### Build dEQP runner
. .gitlab-ci/build-cts-runner.sh
mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/usr/bin
@ -63,6 +113,7 @@ rm -rf /renderdoc
############### Cross-build kernel
KERNEL_URL="https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.5-panfrost-fixes/linux-v5.5-panfrost-fixes.tar.gz"
mkdir -p kernel
wget -qO- ${KERNEL_URL} | tar -xz --strip-components=1 -C kernel
pushd kernel
@ -73,19 +124,6 @@ for image in ${KERNEL_IMAGE_NAME}; do
done
cp ${DEVICE_TREES} /lava-files/.
if [[ ${DEBIAN_ARCH} = "arm64" ]] && which mkimage > /dev/null; then
make Image.lzma
mkimage \
-f auto \
-A arm \
-O linux \
-d arch/arm64/boot/Image.lzma \
-C lzma\
-b arch/arm64/boot/dts/qcom/sdm845-cheza-r3.dtb \
/lava-files/cheza-kernel
fi
popd
rm -rf kernel
@ -105,37 +143,22 @@ set -e
cp .gitlab-ci/create-rootfs.sh /lava-files/rootfs-${DEBIAN_ARCH}/.
chroot /lava-files/rootfs-${DEBIAN_ARCH} sh /create-rootfs.sh
rm /lava-files/rootfs-${DEBIAN_ARCH}/create-rootfs.sh
pushd /lava-files/rootfs-${DEBIAN_ARCH}
find -H | cpio -H newc -o | gzip -c - > /lava-files/lava-rootfs.cpio.gz
popd
rm -rf /lava-files/rootfs-${DEBIAN_ARCH}
if [ ${DEBIAN_ARCH} = arm64 ]; then
# Pull down a specific build of qcomlt/release/qcomlt-5.4 8c79b3d12355
# ("Merge tag 'v5.4.23' into release/qcomlt-5.4"), where I used the
# .config from
# http://snapshots.linaro.org/96boards/dragonboard820c/linaro/debian/457/config-5.4.0-qcomlt-arm64
# with the following merged in:
#
# CONFIG_DRM=y
# CONFIG_DRM_MSM=y
# CONFIG_ATL1C=y
#
# Reason: 5.5 has a big stack of oopses and warns on db820c. 4.14-5.4
# linaro kernel binaries (see above .config link) have these as modules
# and distributed the modules only in the debian system, not the initrd,
# so they're very hard to extract (involving simg2img and loopback
# mounting). 4.11 is missing d72fea538fe6 ("drm/msm: Fix the check for
# the command size") so it can't actually run fredreno. qcomlt-4.14 is
# unstable at boot (~10% instaboot rate). The 5.4 qcomlt kernel with msm
# built in seems like the easiest way to go.
wget https://people.freedesktop.org/~anholt/qcomlt-5.4-msm-build/Image.gz -O Image.gz \
-O /lava-files/db820c-kernel
wget https://people.freedesktop.org/~anholt/qcomlt-5.4-msm-build/apq8096-db820c.dtb \
-O /lava-files/db820c.dtb
ls -lh /lava-files/
# Make a gzipped copy of the Image for db410c.
gzip -k /lava-files/Image
. .gitlab-ci/container/container_post_build.sh
############### Upload the files!
ci-fairy minio login $CI_JOB_JWT
for f in $(ls /lava-files/); do
ci-fairy minio cp /lava-files/$f \
minio://${MINIO_PATH}/$f
done
touch /lava-files/done
ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
# Add missing a630 firmware, added to debian packge in apr 2020
wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/qcom/a630_gmu.bin \
-O /lava-files/rootfs-arm64/lib/firmware/qcom/a630_gmu.bin
wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/qcom/a630_sqe.fw \
-O /lava-files/rootfs-arm64/lib/firmware/qcom/a630_sqe.fw
fi

View File

@ -13,7 +13,6 @@ apt-get -y install --no-install-recommends \
libdrm-nouveau2 \
libx11-6 \
libx11-xcb1 \
firmware-qcom-media \
netcat-openbsd \
python3 \
libpython3.7 \
@ -38,19 +37,13 @@ exec sh
EOF
chmod +x /init
mkdir -p /lib/firmware/rtl_nic
wget https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/rtl_nic/rtl8153a-3.fw -O /lib/firmware/rtl_nic/rtl8153a-3.fw
#######################################################################
# Strip the image to a small minimal system without removing the debian
# toolchain.
# xz compress firmware so it doesn't waste RAM at runtime. Except db820c's
# GPU firmware, due to using a precompiled kernel without compression support.
# xz compress firmware so it doesn't waste RAM at runtime.
find /lib/firmware -type f -print0 | \
grep -vz a530 | \
xargs -0r -P4 -n4 xz -T1 -C crc32
ln -s /lib/firmware/qcom/a530* /lib/firmware/
# Copy timezone file and remove tzdata package
rm -rf /etc/localtime
@ -124,7 +117,6 @@ UNNEEDED_PACKAGES="apt libapt-pkg6.0 "\
"passwd "\
"libsemanage1 libsemanage-common "\
"libsepol1 "\
"gzip "\
"gpgv "\
"hostname "\
"adduser "\

View File

@ -3,11 +3,13 @@
from jinja2 import Environment, FileSystemLoader
import argparse
import os
import datetime
parser = argparse.ArgumentParser()
parser.add_argument("--template")
parser.add_argument("--pipeline-info")
parser.add_argument("--base-artifacts-url")
parser.add_argument("--mesa-url")
parser.add_argument("--device-type")
parser.add_argument("--kernel-image-name")
parser.add_argument("--kernel-image-type", nargs='?', default="")
@ -16,7 +18,6 @@ parser.add_argument("--boot-method")
parser.add_argument("--lava-tags", nargs='?', default="")
parser.add_argument("--env-vars", nargs='?', default="")
parser.add_argument("--deqp-version")
parser.add_argument("--arch")
parser.add_argument("--ci-node-index")
parser.add_argument("--ci-node-total")
args = parser.parse_args()
@ -29,6 +30,7 @@ env_vars = "%s CI_NODE_INDEX=%s CI_NODE_TOTAL=%s" % (args.env_vars, args.ci_node
values = {}
values['pipeline_info'] = args.pipeline_info
values['base_artifacts_url'] = args.base_artifacts_url
values['mesa_url'] = args.mesa_url
values['device_type'] = args.device_type
values['kernel_image_name'] = args.kernel_image_name
values['kernel_image_type'] = args.kernel_image_type
@ -37,7 +39,10 @@ values['boot_method'] = args.boot_method
values['tags'] = args.lava_tags
values['env_vars'] = env_vars
values['deqp_version'] = args.deqp_version
values['arch'] = args.arch
# We need a sane date to check certificates, but don't want to wait to get
# time from the network after boot.
values['date'] = datetime.datetime.now().strftime("%Y%m%d %H%M")
f = open('lava-deqp.yml', "w")
f.write(template.render(values))

View File

@ -21,13 +21,6 @@ actions:
- deploy:
timeout:
minutes: 10
{% if boot_method == "fastboot" %}
{# fastboot has just one boot image with the dtb and rootfs packed in #}
to: fastboot
images:
boot:
url: {{ base_artifacts_url }}/{{ kernel_image_name }}
{% else %}
to: tftp
kernel:
url: {{ base_artifacts_url }}/{{ kernel_image_name }}
@ -35,11 +28,10 @@ actions:
{{ kernel_image_type }}
{% endif %}
ramdisk:
url: {{ base_artifacts_url }}/lava-rootfs-{{ arch }}.cpio.gz
url: {{ base_artifacts_url }}/lava-rootfs.cpio.gz
compression: gz
dtb:
url: {{ base_artifacts_url }}/{{ device_type }}.dtb
{% endif %}
os: oe
- boot:
timeout:
@ -78,6 +70,9 @@ actions:
- mount -t devtmpfs none /dev || echo possibly already mounted
- mkdir -p /dev/pts
- mount -t devpts devpts /dev/pts
- echo "nameserver 8.8.8.8" > /etc/resolv.conf
- date "+%Y%m%d %H%M" -s "{{ date }}"
{% if env_vars %}
- export {{ env_vars }}
@ -86,6 +81,8 @@ actions:
# deqp-runner.sh assumes some stuff is in pwd
- cd /
- wget -S --progress=dot:giga -O- {{ mesa_url }} | tar -xz
- export DEQP_NO_SAVE_RESULTS=1
- 'export DEQP_RUNNER_OPTIONS="--shuffle false"'
- export DEQP_EXPECTED_FAILS=deqp-{{ gpu_version }}-fails.txt

View File

@ -1,3 +1,30 @@
variables:
DISTRIBUTION_TAG: "2020-07-06"
.kernel+rootfs:
stage: container-2
variables:
GIT_STRATEGY: fetch
script:
- DEBIAN_ARCH=${DEBIAN_ARCH} .gitlab-ci/container/lava_arm.sh
rules:
- when: always
kernel+rootfs_arm64:
extends:
- .use-arm_build
- .kernel+rootfs
tags:
- aarch64
variables:
DEBIAN_ARCH: "arm64"
kernel+rootfs_armhf:
extends:
- kernel+rootfs_arm64
variables:
DEBIAN_ARCH: "armhf"
.lava-test:
extends:
- .ci-run-policy
@ -5,22 +32,17 @@
GIT_STRATEGY: none # testing doesn't build anything from source
ENV_VARS: "DEQP_PARALLEL=6"
DEQP_VERSION: gles2
ARTIFACTS_URL: "https://minio-packet.freedesktop.org/mesa-lava/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}"
MESA_URL: "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"
script:
- mkdir -p /srv/${FILES_HOST_NAME}/$CI_JOB_ID/
- cp /lava-files/${KERNEL_IMAGE_NAME} /srv/${FILES_HOST_NAME}/$CI_JOB_ID/.
- cp /lava-files/${DEVICE_TYPE}.dtb /srv/${FILES_HOST_NAME}/$CI_JOB_ID/.
- tar -C /lava-files/rootfs-${ARCH} -xf artifacts/install.tar
- pushd /lava-files/rootfs-${ARCH}
- find -H | cpio -H newc -o | gzip -c - > /srv/${FILES_HOST_NAME}/$CI_JOB_ID/lava-rootfs-${ARCH}.cpio.gz
- popd
- >
artifacts/generate_lava.py \
--template artifacts/lava-deqp.yml.jinja2 \
--pipeline-info "$CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
--base-artifacts-url ${FILES_HOST_URL}/$CI_JOB_ID \
--base-artifacts-url ${ARTIFACTS_URL} \
--mesa-url ${MESA_URL} \
--device-type ${DEVICE_TYPE} \
--env-vars "${ENV_VARS}" \
--arch ${ARCH} \
--deqp-version ${DEQP_VERSION} \
--kernel-image-name ${KERNEL_IMAGE_NAME} \
--kernel-image-type "${KERNEL_IMAGE_TYPE}" \
@ -38,8 +60,6 @@
- result=`lavacli results $lava_job_id 0_deqp deqp | head -1`
- echo $result
- '[[ "$result" == "pass" ]]'
after_script:
- rm -rf /srv/${FILES_HOST_NAME}/$CI_JOB_ID/
artifacts:
name: "mesa_${CI_JOB_NAME}"
when: always
@ -58,7 +78,7 @@
dependencies:
- meson-armhf
needs:
- arm_build
- kernel+rootfs_armhf
- meson-armhf
.lava-test:arm64:
@ -73,7 +93,7 @@
dependencies:
- meson-arm64
needs:
- arm_build
- kernel+rootfs_arm64
- meson-arm64
panfrost-t720-gles2:arm64:
@ -84,8 +104,6 @@ panfrost-t720-gles2:arm64:
DEVICE_TYPE: sun50i-h6-pine-h64
GPU_VERSION: panfrost-t720
ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T720"
FILES_HOST_NAME: "mesa-ci-lava-files.freedesktop.org"
FILES_HOST_URL: "https://mesa-ci-lava-files.freedesktop.org"
tags:
- mesa-ci-aarch64-lava-collabora
@ -99,8 +117,6 @@ panfrost-t760-gles2:armhf:
BOOT_METHOD: depthcharge
KERNEL_IMAGE_TYPE: ""
ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T760"
FILES_HOST_NAME: "mesa-ci-lava-files.freedesktop.org"
FILES_HOST_URL: "https://mesa-ci-lava-files.freedesktop.org"
tags:
- mesa-ci-aarch64-lava-collabora
@ -114,8 +130,6 @@ panfrost-t860-gles2:arm64:
BOOT_METHOD: depthcharge
KERNEL_IMAGE_TYPE: ""
ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T860"
FILES_HOST_NAME: "mesa-ci-lava-files.freedesktop.org"
FILES_HOST_URL: "https://mesa-ci-lava-files.freedesktop.org"
tags:
- mesa-ci-aarch64-lava-collabora
@ -132,8 +146,6 @@ panfrost-t860-gles3:arm64:
ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T860"
CI_NODE_INDEX: 1
CI_NODE_TOTAL: 5
FILES_HOST_NAME: "mesa-ci-lava-files.freedesktop.org"
FILES_HOST_URL: "https://mesa-ci-lava-files.freedesktop.org"
tags:
- mesa-ci-aarch64-lava-collabora

View File

@ -47,8 +47,13 @@ mkdir -p artifacts/
tar -cf artifacts/install.tar install
# If the container has LAVA stuff, prepare the artifacts for LAVA jobs
if [ -d /lava-files ]; then
if [ -n "$UPLOAD_FOR_LAVA" ]; then
# Pass needed files to the test stage
cp $CI_PROJECT_DIR/.gitlab-ci/generate_lava.py artifacts/.
cp $CI_PROJECT_DIR/.gitlab-ci/lava-deqp.yml.jinja2 artifacts/.
gzip -c artifacts/install.tar > mesa-${DEBIAN_ARCH}.tar.gz
ci-fairy minio login $CI_JOB_JWT
MINIO_PATH=minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
ci-fairy minio cp mesa-${DEBIAN_ARCH}.tar.gz minio://${MINIO_PATH}/mesa-${DEBIAN_ARCH}.tar.gz
fi