diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93f4cbf89e8a7..19a29871e8b2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: rm download-git-cache.sh set +o xtrace CI_JOB_JWT_FILE: /minio_jwt - MINIO_HOST: minio-packet.freedesktop.org + MINIO_HOST: s3.freedesktop.org # per-pipeline artifact storage on MinIO PIPELINE_ARTIFACTS_BASE: ${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID} # per-job artifact storage on MinIO @@ -264,9 +264,7 @@ make git archive: # compress the current folder - tar -cvzf ../$CI_PROJECT_NAME.tar.gz . - # login with the JWT token file - - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" - - ci-fairy minio cp ../$CI_PROJECT_NAME.tar.gz minio://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz + - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz # Sanity checks of MR settings and commit logs diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh index ad6528d1948e7..a0718bf37fc65 100755 --- a/.gitlab-ci/common/init-stage2.sh +++ b/.gitlab-ci/common/init-stage2.sh @@ -150,8 +150,7 @@ cleanup # upload artifacts if [ -n "$MINIO_RESULTS_UPLOAD" ]; then tar --zstd -cf results.tar.zst results/; - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"; - ci-fairy minio cp results.tar.zst minio://"$MINIO_RESULTS_UPLOAD"/results.tar.zst; + ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" results.tar.zst https://"$MINIO_RESULTS_UPLOAD"/results.tar.zst; fi # We still need to echo the hwci: mesa message, as some scripts rely on it, such diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index 59276ec6c31d0..8cfcc39cde425 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -267,7 +267,6 @@ popd . .gitlab-ci/container/container_post_build.sh ############### Upload the files! -ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" FILES_TO_UPLOAD="lava-rootfs.tar.zst \ $KERNEL_IMAGE_NAME" @@ -276,9 +275,9 @@ if [[ -n $DEVICE_TREES ]]; then fi for f in $FILES_TO_UPLOAD; do - ci-fairy minio cp /lava-files/$f \ - minio://${MINIO_PATH}/$f + ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \ + https://${MINIO_PATH}/$f done touch /lava-files/done -ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done +ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${MINIO_PATH}/done diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index 8cce9baef3ab0..d0df3c72319ef 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -26,8 +26,7 @@ cat results/job-rootfs-overlay/set-job-env-vars.sh echo -e "\e[0Ksection_end:$(date +%s):variables\r\e[0K" tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ . -ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" -ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}" +ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}" touch results/lava.log tail -f results/lava.log & diff --git a/.gitlab-ci/piglit/piglit-traces.sh b/.gitlab-ci/piglit/piglit-traces.sh index ddb2fea6f92b7..089b7449dd2f3 100755 --- a/.gitlab-ci/piglit/piglit-traces.sh +++ b/.gitlab-ci/piglit/piglit-traces.sh @@ -3,7 +3,7 @@ set -ex INSTALL=$(realpath -s "$PWD"/install) -MINIO_ARGS="--credentials=/tmp/.minio_credentials" +MINIO_ARGS="--token-file ${CI_JOB_JWT_FILE}" RESULTS=$(realpath -s "$PWD"/results) mkdir -p "$RESULTS" @@ -137,8 +137,8 @@ replay_minio_upload_images() { __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}" fi - ci-fairy minio cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \ - "minio://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" + ci-fairy s3cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \ + "https://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" done } @@ -173,8 +173,6 @@ if [ "$RUN_CMD_WRAPPER" ]; then RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e" fi -ci-fairy minio login $MINIO_ARGS --token-file "${CI_JOB_JWT_FILE}" - # The replayer doesn't do any size or checksum verification for the traces in # the replayer db, so if we had to restart the system due to intermittent device # errors (or tried to cache replayer-db between runs, which would be nice to diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index 3933a2c984fd5..bd098d5f7be28 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -54,6 +54,5 @@ if [ -n "$MINIO_ARTIFACT_NAME" ]; then # Pass needed files to the test stage MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.zst" zstd artifacts/install.tar -o ${MINIO_ARTIFACT_NAME} - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" - ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME} + ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${MINIO_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME} fi diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index 1b15116ac07e7..e82b7e0045aa8 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -87,7 +87,7 @@ rustfmt: exclude: - results/*.shader_cache variables: - PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-public --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE} + PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${CI_JOB_JWT_FILE} script: - echo -e "\e[0Ksection_start:$(date +%s):variables[collapsed=true]\r\e[0KVariables passed through:" - install/common/generate-env.sh diff --git a/docs/ci/bare-metal.rst b/docs/ci/bare-metal.rst index f8c51b28999c3..544f4186da15e 100644 --- a/docs/ci/bare-metal.rst +++ b/docs/ci/bare-metal.rst @@ -219,12 +219,12 @@ Enable the site and restart nginx: sudo service nginx restart # First download will hit the internet - wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace + wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace # Second download should be cached. - wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace + wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace Now, set ``download-url`` in your ``traces-*.yml`` entry to something like -``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public`` +``http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public`` and you should have cached downloads for traces. Add it to ``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you can use it for cached artifact downloads instead of going all the way to diff --git a/src/amd/ci/traces-amd.yml b/src/amd/ci/traces-amd.yml index e2f702574c14b..63da7006995ea 100644 --- a/src/amd/ci/traces-amd.yml +++ b/src/amd/ci/traces-amd.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/" traces: unigine/heaven-scene1-low-d3d11.trace-dxgi: diff --git a/src/broadcom/ci/traces-broadcom.yml b/src/broadcom/ci/traces-broadcom.yml index 07baf344f2ce6..1f79b62395c2b 100644 --- a/src/broadcom/ci/traces-broadcom.yml +++ b/src/broadcom/ci/traces-broadcom.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "http://192.168.40.131:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "http://192.168.40.131:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/" traces: 0ad/0ad-v2.trace: diff --git a/src/freedreno/ci/gitlab-ci.yml b/src/freedreno/ci/gitlab-ci.yml index 13949d440f81c..f15e464c00a47 100644 --- a/src/freedreno/ci/gitlab-ci.yml +++ b/src/freedreno/ci/gitlab-ci.yml @@ -277,7 +277,7 @@ a630-traces-restricted: - .freedreno-rules-restricted variables: PIGLIT_REPLAY_DESCRIPTION_FILE: "/install/restricted-traces-freedreno.yml" - PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE} + PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE} allow_failure: true a630-traces-performance: diff --git a/src/freedreno/ci/restricted-traces-freedreno.yml b/src/freedreno/ci/restricted-traces-freedreno.yml index b5ac491d112b6..9b7913e1dcd79 100644 --- a/src/freedreno/ci/restricted-traces-freedreno.yml +++ b/src/freedreno/ci/restricted-traces-freedreno.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-private/" + download-url: "http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-private/" traces: stellaris/Stellaris-trim--f722-v20201203-v2.trace: diff --git a/src/freedreno/ci/traces-freedreno.yml b/src/freedreno/ci/traces-freedreno.yml index 6e9c267596f29..e60257b4b75b4 100644 --- a/src/freedreno/ci/traces-freedreno.yml +++ b/src/freedreno/ci/traces-freedreno.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/ + download-url: http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/ text: |- NOTE for faster runtime, try to sort tests so the longer running traces diff --git a/src/gallium/drivers/crocus/ci/traces-crocus.yml b/src/gallium/drivers/crocus/ci/traces-crocus.yml index 09af0a2089d28..d777e015aae03 100644 --- a/src/gallium/drivers/crocus/ci/traces-crocus.yml +++ b/src/gallium/drivers/crocus/ci/traces-crocus.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: 0ad/0ad-v2.trace: diff --git a/src/gallium/drivers/i915/ci/traces-i915.yml b/src/gallium/drivers/i915/ci/traces-i915.yml index 26a5aaa278615..f163090dc4e4c 100644 --- a/src/gallium/drivers/i915/ci/traces-i915.yml +++ b/src/gallium/drivers/i915/ci/traces-i915.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: glxgears/glxgears-2-v2.trace: diff --git a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml index 70e621ef25f7e..32fe5e455622b 100644 --- a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml +++ b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: 0ad/0ad-v2.trace: diff --git a/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml b/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml index 05aaf2c9541c9..5e9a21cee3edf 100644 --- a/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml +++ b/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace: diff --git a/src/gallium/drivers/virgl/ci/traces-virgl.yml b/src/gallium/drivers/virgl/ci/traces-virgl.yml index 2be64e826d5f5..f480553bedad0 100644 --- a/src/gallium/drivers/virgl/ci/traces-virgl.yml +++ b/src/gallium/drivers/virgl/ci/traces-virgl.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace: diff --git a/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml b/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml index 3421845a58532..e9e35462af29c 100644 --- a/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml +++ b/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: unigine/sanctuary-d3d9.trace-dxgi: diff --git a/src/intel/ci/traces-iris.yml b/src/intel/ci/traces-iris.yml index 0880d0d6a6856..891f0852d6c26 100644 --- a/src/intel/ci/traces-iris.yml +++ b/src/intel/ci/traces-iris.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/" traces: behdad-glyphy/glyphy-v2.trace: diff --git a/src/panfrost/ci/traces-panfrost.yml b/src/panfrost/ci/traces-panfrost.yml index 968135e633db8..56adcf4593286 100644 --- a/src/panfrost/ci/traces-panfrost.yml +++ b/src/panfrost/ci/traces-panfrost.yml @@ -1,7 +1,7 @@ %YAML 1.2 --- traces-db: - download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/" traces: behdad-glyphy/glyphy-v2.trace: