mirrors
/
mesa
mirror of https://gitlab.freedesktop.org/mesa/mesa
1
0
Fork 0
Code Issues 9 Packages Projects Releases Wiki Activity

CI: convert to use the new S3 server instead of the legacy minio 

We don't need to login anymore, but we can't use plain minio commands
now. `ci-fairy` got a helper as `s3cp` to keep an almost identical
API.

Signed-off-by: Benjamin Tissoires <benjamin.tissoires@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19076>
This commit is contained in:
Benjamin Tissoires 2022-10-13 21:25:38 +02:00 committed by Marge Bot
parent a04ed2f971
commit 67cee534a8
21 changed files with 28 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitlab-ci.yml
View File

@ -8,7 +8,7 @@ variables:
rm download-git-cache.sh
set +o xtrace
CI_JOB_JWT_FILE: /minio_jwt
MINIO_HOST: minio-packet.freedesktop.org
MINIO_HOST: s3.freedesktop.org
# per-pipeline artifact storage on MinIO
PIPELINE_ARTIFACTS_BASE: ${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
# per-job artifact storage on MinIO
@ -264,9 +264,7 @@ make git archive:
# compress the current folder
- tar -cvzf ../$CI_PROJECT_NAME.tar.gz .
# login with the JWT token file
- ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
- ci-fairy minio cp ../$CI_PROJECT_NAME.tar.gz minio://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
- ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
# Sanity checks of MR settings and commit logs

3
.gitlab-ci/common/init-stage2.sh
View File

@ -150,8 +150,7 @@ cleanup
# upload artifacts
if [ -n "$MINIO_RESULTS_UPLOAD" ]; then
tar --zstd -cf results.tar.zst results/;
ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}";
ci-fairy minio cp results.tar.zst minio://"$MINIO_RESULTS_UPLOAD"/results.tar.zst;
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" results.tar.zst https://"$MINIO_RESULTS_UPLOAD"/results.tar.zst;
fi
# We still need to echo the hwci: mesa message, as some scripts rely on it, such

7
.gitlab-ci/container/lava_build.sh
View File

@ -267,7 +267,6 @@ popd
. .gitlab-ci/container/container_post_build.sh
############### Upload the files!
ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
FILES_TO_UPLOAD="lava-rootfs.tar.zst \
$KERNEL_IMAGE_NAME"
@ -276,9 +275,9 @@ if [[ -n $DEVICE_TREES ]]; then
fi
for f in $FILES_TO_UPLOAD; do
ci-fairy minio cp /lava-files/$f \
minio://${MINIO_PATH}/$f
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \
https://${MINIO_PATH}/$f
done
touch /lava-files/done
ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${MINIO_PATH}/done

3
.gitlab-ci/lava/lava-submit.sh
View File

@ -26,8 +26,7 @@ cat results/job-rootfs-overlay/set-job-env-vars.sh
echo -e "\e[0Ksection_end:$(date +%s):variables\r\e[0K"
tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}"
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
touch results/lava.log
tail -f results/lava.log &

8
.gitlab-ci/piglit/piglit-traces.sh
View File

@ -3,7 +3,7 @@
set -ex
INSTALL=$(realpath -s "$PWD"/install)
MINIO_ARGS="--credentials=/tmp/.minio_credentials"
MINIO_ARGS="--token-file ${CI_JOB_JWT_FILE}"
RESULTS=$(realpath -s "$PWD"/results)
mkdir -p "$RESULTS"
@ -137,8 +137,8 @@ replay_minio_upload_images() {
__DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
fi
ci-fairy minio cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
"minio://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
ci-fairy s3cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
"https://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
done
}
@ -173,8 +173,6 @@ if [ "$RUN_CMD_WRAPPER" ]; then
RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
fi
ci-fairy minio login $MINIO_ARGS --token-file "${CI_JOB_JWT_FILE}"
# The replayer doesn't do any size or checksum verification for the traces in
# the replayer db, so if we had to restart the system due to intermittent device
# errors (or tried to cache replayer-db between runs, which would be nice to

3
.gitlab-ci/prepare-artifacts.sh
View File

@ -54,6 +54,5 @@ if [ -n "$MINIO_ARTIFACT_NAME" ]; then
# Pass needed files to the test stage
MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.zst"
zstd artifacts/install.tar -o ${MINIO_ARTIFACT_NAME}
ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${MINIO_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
fi

2
.gitlab-ci/test/gitlab-ci.yml
View File

@ -87,7 +87,7 @@ rustfmt:
exclude:
- results/*.shader_cache
variables:
PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-public --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE}
PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${CI_JOB_JWT_FILE}
script:
- echo -e "\e[0Ksection_start:$(date +%s):variables[collapsed=true]\r\e[0KVariables passed through:"
- install/common/generate-env.sh

6
docs/ci/bare-metal.rst
View File

@ -219,12 +219,12 @@ Enable the site and restart nginx:
sudo service nginx restart
# First download will hit the internet
wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
# Second download should be cached.
wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
Now, set ``download-url`` in your ``traces-*.yml`` entry to something like
``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public``
``http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public``
and you should have cached downloads for traces. Add it to
``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you
can use it for cached artifact downloads instead of going all the way to

2
src/amd/ci/traces-amd.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
unigine/heaven-scene1-low-d3d11.trace-dxgi:

2
src/broadcom/ci/traces-broadcom.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "http://192.168.40.131:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://192.168.40.131:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
0ad/0ad-v2.trace:

2
src/freedreno/ci/gitlab-ci.yml
View File

@ -277,7 +277,7 @@ a630-traces-restricted:
- .freedreno-rules-restricted
variables:
PIGLIT_REPLAY_DESCRIPTION_FILE: "/install/restricted-traces-freedreno.yml"
PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE}
PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE}
allow_failure: true
a630-traces-performance:

2
src/freedreno/ci/restricted-traces-freedreno.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-private/"
download-url: "http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-private/"
traces:
stellaris/Stellaris-trim--f722-v20201203-v2.trace:

2
src/freedreno/ci/traces-freedreno.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/
download-url: http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/
text: |-
NOTE for faster runtime, try to sort tests so the longer running traces

2
src/gallium/drivers/crocus/ci/traces-crocus.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
0ad/0ad-v2.trace:

2
src/gallium/drivers/i915/ci/traces-i915.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
glxgears/glxgears-2-v2.trace:

2
src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
0ad/0ad-v2.trace:

2
src/gallium/drivers/virgl/ci/traces-virgl-iris.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace:

2
src/gallium/drivers/virgl/ci/traces-virgl.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace:

2
src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
unigine/sanctuary-d3d9.trace-dxgi:

2
src/intel/ci/traces-iris.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
behdad-glyphy/glyphy-v2.trace:

2
src/panfrost/ci/traces-panfrost.yml
View File

@ -1,7 +1,7 @@
%YAML 1.2
---
traces-db:
download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
behdad-glyphy/glyphy-v2.trace: