ci: Use ci-fairy minio login via token file

For every CI job, put JWT content into a file and unset CI_JOB_JWT
environment var
=======

* virgl jobs:
	- Share JWT token file to crosvm instance
	- Keep using `export -p` due to high complexity in the scripts
	  of these jobs. At least, the CI_JOB_JWT will not be leaked,
	  since it is being unset at the `before_script` phase of each
	  Mesa CI job.

* iris jobs: Update lava_job_submitter to take token file as argument
	- generate-env with CI_JOB_JWT_TOKEN_FILE
	- create token file during baremetal init stage

* baremetal jobs: Copy token file to bare-metal NFS

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Reviewed-by: Cristian Ciocaltea <cristian.ciocaltea@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/14004>

.gitlab-ci.yml

@@ -16,6 +16,22 @@ variables:
   # running on a particular CI farm (ie. for outages, etc):
   FD_FARM: "online"
 
+default:
+  before_script:
+    - echo -e "\e[0Ksection_start:$(date +%s):unset_env_vars_section[collapsed=true]\r\e[0KUnsetting vulnerable environment variables"
+    - export CI_JOB_JWT_FILE="${CI_JOB_JWT_FILE:-$(mktemp)}"
+    - echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}"
+    - unset CI_JOB_JWT
+    - echo -e "\e[0Ksection_end:$(date +%s):unset_env_vars_section\r\e[0K"
+
+  after_script:
+    - >
+      set +x
+
+      test -e "${CI_JOB_JWT_FILE}" &&
+      export CI_JOB_JWT="$(<${CI_JOB_JWT_FILE})" &&
+      rm "${CI_JOB_JWT_FILE}"
+
 include:
   - project: 'freedesktop/ci-templates'
     ref: 34f4ade99434043f88e164933f570301fd18b125

.gitlab-ci/bare-metal/rootfs-setup.sh

@@ -8,15 +8,20 @@ mkdir -p $rootfs_dst/results
 cp $BM/bm-init.sh $rootfs_dst/init
 cp $CI_COMMON/init*.sh $rootfs_dst/
 
+# Make JWT token available as file in the bare-metal storage to enable access
+# to MinIO
+cp "${CI_JOB_JWT_FILE}" "${rootfs_dst}${CI_JOB_JWT_FILE}"
+
 cp $CI_COMMON/capture-devcoredump.sh $rootfs_dst/
 
 set +x
+
 # Pass through relevant env vars from the gitlab job to the baremetal init script
 "$CI_COMMON"/generate-env.sh > $rootfs_dst/set-job-env-vars.sh
 chmod +x $rootfs_dst/set-job-env-vars.sh
 echo "Variables passed through:"
 cat $rootfs_dst/set-job-env-vars.sh
-echo "export CI_JOB_JWT=${CI_JOB_JWT@Q}" >> $rootfs_dst/set-job-env-vars.sh
+
 set -x
 
 # Add the Mesa drivers we built, and make a consistent symlink to them.

.gitlab-ci/common/generate-env.sh

@@ -7,6 +7,7 @@ for var in \
     CI_COMMIT_BRANCH \
     CI_COMMIT_TITLE \
     CI_JOB_ID \
+    CI_JOB_JWT_FILE \
     CI_JOB_URL \
     CI_MERGE_REQUEST_SOURCE_BRANCH_NAME \
     CI_MERGE_REQUEST_TITLE \
@@ -20,6 +21,9 @@ for var in \
     CI_PROJECT_ROOT_NAMESPACE \
     CI_RUNNER_DESCRIPTION \
     CI_SERVER_URL \
+    CROSVM_GALLIUM_DRIVER \
+    CROSVM_GPU_ARGS \
+    CROSVM_TEST_SCRIPT \
     DEQP_CASELIST_FILTER \
     DEQP_CASELIST_INV_FILTER \
     DEQP_CONFIG \
@@ -29,6 +33,7 @@ for var in \
     DEQP_RESULTS_DIR \
     DEQP_RUNNER_OPTIONS \
     DEQP_SUITE \
+    DEQP_TEMP_DIR \
     DEQP_VARIANT \
     DEQP_VER \
     DEQP_WIDTH \
@@ -40,6 +45,7 @@ for var in \
     FDO_UPSTREAM_REPO \
     FD_MESA_DEBUG \
     FLAKES_CHANNEL \
+    GALLIUM_DRIVER \
     GPU_VERSION \
     GTEST \
     GTEST_FAILS \
@@ -55,10 +61,11 @@ for var in \
     JOB_ARTIFACTS_BASE \
     JOB_RESULTS_PATH \
     JOB_ROOTFS_OVERLAY_PATH \
+    LD_LIBRARY_PATH \
     MESA_BUILD_PATH \
-    MESA_GL_VERSION_OVERRIDE \
-    MESA_GLSL_VERSION_OVERRIDE \
     MESA_GLES_VERSION_OVERRIDE \
+    MESA_GLSL_VERSION_OVERRIDE \
+    MESA_GL_VERSION_OVERRIDE \
     MESA_VK_IGNORE_CONFORMANCE_WARNING \
     MINIO_HOST \
     NIR_VALIDATE \
@@ -71,11 +78,11 @@ for var in \
     PIGLIT_PLATFORM \
     PIGLIT_PROFILES \
     PIGLIT_REPLAY_ARTIFACTS_BASE_URL \
-    PIGLIT_REPLAY_SUBCOMMAND \
     PIGLIT_REPLAY_DESCRIPTION_FILE \
     PIGLIT_REPLAY_DEVICE_NAME \
     PIGLIT_REPLAY_EXTRA_ARGS \
     PIGLIT_REPLAY_REFERENCE_IMAGES_BASE \
+    PIGLIT_REPLAY_SUBCOMMAND \
     PIGLIT_REPLAY_UPLOAD_TO_MINIO \
     PIGLIT_RESULTS \
     PIGLIT_TESTS \

.gitlab-ci/common/init-stage2.sh

@@ -71,7 +71,7 @@ fi
 MINIO=$(cat /proc/cmdline | tr ' ' '\n' | grep minio_results | cut -d '=' -f 2 || true)
 if [ -n "$MINIO" ]; then
   tar -czf results.tar.gz results/;
-  ci-fairy minio login "$CI_JOB_JWT";
+  ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}";
   ci-fairy minio cp results.tar.gz minio://"$MINIO"/results.tar.gz;
 fi
 

.gitlab-ci/container/lava_build.sh

@@ -205,7 +205,7 @@ popd
 . .gitlab-ci/container/container_post_build.sh
 
 ############### Upload the files!
-ci-fairy minio login $CI_JOB_JWT
+ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
 FILES_TO_UPLOAD="lava-rootfs.tgz \
                  $KERNEL_IMAGE_NAME"
 

.gitlab-ci/crosvm-runner.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-set -e
+set -ex
 
 # This script can be called concurrently, pass arguments and env in a per-instance tmp dir
 export DEQP_TEMP_DIR=`mktemp -d /tmp.XXXXXXXXXX`

.gitlab-ci/lava/lava-submit.sh

@@ -22,7 +22,7 @@ cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/
 artifacts/ci-common/generate-env.sh > results/job-rootfs-overlay/set-job-env-vars.sh
 
 tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
-ci-fairy minio login "${CI_JOB_JWT}"
+ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
 ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}"
 
 touch results/lava.log
@@ -39,7 +39,7 @@ artifacts/lava/lava_job_submitter.py \
 	--ci-project-dir ${CI_PROJECT_DIR} \
 	--device-type ${DEVICE_TYPE} \
 	--dtb ${DTB} \
-	--jwt "${CI_JOB_JWT}" \
+	--jwt-file "${CI_JOB_JWT_FILE}" \
 	--kernel-image-name ${KERNEL_IMAGE_NAME} \
 	--kernel-image-type "${KERNEL_IMAGE_TYPE}" \
 	--boot-method ${BOOT_METHOD} \

.gitlab-ci/lava/lava_job_submitter.py

@@ -25,16 +25,16 @@
 """Send a job to LAVA, track it and collect log back"""
 
 import argparse
-import lavacli
-import os
+import pathlib
 import sys
 import time
 import traceback
 import urllib.parse
 import xmlrpc
-import yaml
-
 from datetime import datetime, timedelta
+
+import lavacli
+import yaml
 from lavacli.utils import loader
 
 # Timeout in minutes to decide if the device from the dispatched LAVA job has
@@ -59,6 +59,18 @@ def fatal_err(msg):
     print_log(msg)
     sys.exit(1)
 
+
+def hide_sensitive_data(yaml_data, hide_tag="HIDEME"):
+    out_data = ""
+
+    for line in yaml_data.splitlines(True):
+        if hide_tag in line:
+            continue
+        out_data += line
+
+    return out_data
+
+
 def generate_lava_yaml(args):
     # General metadata and permissions, plus also inexplicably kernel arguments
     values = {
@@ -140,15 +152,22 @@ def generate_lava_yaml(args):
     #   - fetch and unpack per-job environment from lava-submit.sh
     #   - exec .gitlab-ci/common/init-stage2.sh 
     init_lines = []
+
     with open(args.first_stage_init, 'r') as init_sh:
       init_lines += [ x.rstrip() for x in init_sh if not x.startswith('#') and x.rstrip() ]
+
+    with open(args.jwt_file) as jwt_file:
+        init_lines += [
+            "set +x",
+            f'echo -n "{jwt_file.read()}" > "{args.jwt_file}"  # HIDEME',
+            "set -x",
+        ]
+
     init_lines += [
       'mkdir -p {}'.format(args.ci_project_dir),
       'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.mesa_build_url, args.ci_project_dir),
       'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url),
-      'set +x',
-      'export CI_JOB_JWT="{}"'.format(args.jwt),
-      'set -x',
+      f'echo "export CI_JOB_JWT_FILE={args.jwt_file}" >> /set-job-env-vars.sh',
       'exec /init-stage2.sh',
     ]
     test['definitions'][0]['repository']['run']['steps'] = init_lines
@@ -285,9 +304,7 @@ def main(args):
     yaml_file = generate_lava_yaml(args)
 
     if args.dump_yaml:
-        censored_args = args
-        censored_args.jwt = "jwt-hidden"
-        print(generate_lava_yaml(censored_args))
+        print(hide_sensitive_data(generate_lava_yaml(args)))
 
     if args.validate_only:
         ret = validate_job(proxy, yaml_file)
@@ -318,13 +335,7 @@ def main(args):
         if get_job_results(proxy,  job_id, "0_mesa", "mesa") == True:
              break
 
-
-if __name__ == '__main__':
-    # given that we proxy from DUT -> LAVA dispatcher -> LAVA primary -> us ->
-    # GitLab runner -> GitLab primary -> user, safe to say we don't need any
-    # more buffering
-    sys.stdout.reconfigure(line_buffering=True)
-    sys.stderr.reconfigure(line_buffering=True)
+def create_parser():
     parser = argparse.ArgumentParser("LAVA job submitter")
 
     parser.add_argument("--pipeline-info")
@@ -341,11 +352,22 @@ if __name__ == '__main__':
     parser.add_argument("--kernel-image-type", nargs='?', default="")
     parser.add_argument("--boot-method")
     parser.add_argument("--lava-tags", nargs='?', default="")
-    parser.add_argument("--jwt")
+    parser.add_argument("--jwt-file", type=pathlib.Path)
     parser.add_argument("--validate-only", action='store_true')
     parser.add_argument("--dump-yaml", action='store_true')
     parser.add_argument("--visibility-group")
 
+    return parser
+
+if __name__ == "__main__":
+    # given that we proxy from DUT -> LAVA dispatcher -> LAVA primary -> us ->
+    # GitLab runner -> GitLab primary -> user, safe to say we don't need any
+    # more buffering
+    sys.stdout.reconfigure(line_buffering=True)
+    sys.stderr.reconfigure(line_buffering=True)
+
+    parser = create_parser()
+
     parser.set_defaults(func=main)
     args = parser.parse_args()
     args.func(args)

.gitlab-ci/piglit/run.sh

@@ -201,7 +201,7 @@ FAILURE_MESSAGE=$(printf "%s" "Unexpected change in results:")
 
 if [ "x$PIGLIT_PROFILES" = "xreplay" ] \
        && [ ${PIGLIT_REPLAY_UPLOAD_TO_MINIO:-0} -eq 1 ]; then
-    ci-fairy minio login $MINIO_ARGS $CI_JOB_JWT
+    ci-fairy minio login $MINIO_ARGS --token-file "${CI_JOB_JWT_FILE}"
 fi
 
 eval $RUN_CMD

.gitlab-ci/prepare-artifacts.sh

@@ -52,6 +52,6 @@ if [ -n "$MINIO_ARTIFACT_NAME" ]; then
     # Pass needed files to the test stage
     MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.gz"
     gzip -c artifacts/install.tar > ${MINIO_ARTIFACT_NAME}
-    ci-fairy minio login $CI_JOB_JWT
+    ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
     ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
 fi

src/freedreno/ci/gitlab-ci.yml

@@ -265,7 +265,7 @@ a630-traces-restricted:
     - .freedreno-rules-restricted
   variables:
     PIGLIT_REPLAY_DESCRIPTION_FILE: "/install/restricted-traces-freedreno.yml"
-    PIGLIT_REPLAY_EXTRA_ARGS:  --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt=${CI_JOB_JWT}
+    PIGLIT_REPLAY_EXTRA_ARGS:  --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE}
   allow_failure: true
 
 a630-traces-performance: