From 2e2edaa89b4cfd935f1dde28e81bbcacf39ad15a Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Thu, 21 Jan 2021 13:55:02 -0800 Subject: [PATCH] docs/ci: Document setting up the http cache for traces. Reviewed-by: Daniel Stone Part-of: --- docs/ci/bare-metal.rst | 38 ++++++++++++++++++++++++++++++++++ docs/ci/fdo-cache | 44 ++++++++++++++++++++++++++++++++++++++++ docs/ci/uri-caching.conf | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 docs/ci/fdo-cache create mode 100644 docs/ci/uri-caching.conf diff --git a/docs/ci/bare-metal.rst b/docs/ci/bare-metal.rst index be5952da21c..3ea57749417 100644 --- a/docs/ci/bare-metal.rst +++ b/docs/ci/bare-metal.rst @@ -128,3 +128,41 @@ that gets used to start a nginx server. Once you've updated your runners' configs, restart with ``sudo service gitlab-runner restart`` + +Caching downloads +----------------- + +To improve the runtime for downloading traces during traces job runs, you will +want a pass-through HTTP cache. On your runner box, install nginx: + +.. code-block:: console + sudo apt install nginx libnginx-mod-http-lua + +Add the server setup files: + +.. literalinclude: fdo-cache: + :name: /etc/nginx/sites-available/fdo-cache + +.. literalinclude: uri-caching.conf: + :name: /etc/nginx/sites-available/snippets/uri-caching.conf + +Edit the listener addresses in fdo-cache to suit the ethernet interface that +your devices are on. + +Enable the site and restart nginx: + +.. code-block:: console + sudo ln -s /etc/nginx/sites-available/fdo-cache /etc/nginx/sites-enabled/fdo-cache + sudo service nginx restart + + # First download will hit the internet + wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace + # Second download should be cached. + wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace + +Now, set ``download-url`` in your ``traces-*.yml`` entry to something like +``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public`` +and you should have cached downloads for traces. Add it to +``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you +can use it for cached artifact downloads instead of going all the way to +freedesktop.org on each job. diff --git a/docs/ci/fdo-cache b/docs/ci/fdo-cache new file mode 100644 index 00000000000..922cd76b240 --- /dev/null +++ b/docs/ci/fdo-cache @@ -0,0 +1,44 @@ +proxy_cache_path /var/cache/nginx/ levels=1:2 keys_zone=my_cache:10m max_size=24g inactive=48h use_temp_path=off; + +server { + listen 10.42.0.1:8888 default_server; + listen 127.0.0.1:8888 default_server; + listen [::]:8888 default_server; + resolver 8.8.8.8; + + root /var/www/html; + + # Add index.php to the list if you are using PHP + index index.html index.htm index.nginx-debian.html; + + server_name _; + + add_header X-GG-Cache-Status $upstream_cache_status; + proxy_cache my_cache; + + location /cache_gitlab_artifacts { + internal; + # Gitlabs http server puts everything as no-cache even though + # the artifacts URLS don't change. So enforce a long validity + # time and ignore the headers that defeat caching + proxy_cache_valid 200 48h; + proxy_ignore_headers Cache-Control Set-Cookie; + include snippets/uri-caching.conf; + } + + location /cache { + # special case gitlab artifacts + if ($arg_uri ~* /.*gitlab.*artifacts(\/|%2F)raw/ ) { + rewrite ^ /cache_gitlab_artifacts; + } + # Set a really low validity together with cache revalidation; Our goal + # for caching isn't to lower the number of http requests but to + # lower the amount of data transfer. Also for some test + # scenarios (typical manual tests) the file at a given url + # might get modified so avoid confusion by ensuring + # revalidations happens often. + proxy_cache_valid 200 10s; + proxy_cache_revalidate on; + include snippets/uri-caching.conf; + } +} diff --git a/docs/ci/uri-caching.conf b/docs/ci/uri-caching.conf new file mode 100644 index 00000000000..c360c55c33a --- /dev/null +++ b/docs/ci/uri-caching.conf @@ -0,0 +1,35 @@ +set $authorization ''; + +set_by_lua $proxyuri ' + unescaped = ngx.unescape_uri(ngx.var.arg_uri); + it, err = ngx.re.match(unescaped, "(https?://)(.*@)?([^/]*)(/.*)?"); + if not it then + -- Hack to cause nginx to return 404 + return "http://localhost/404" + end + + scheme = it[1]; + authstring = it[2]; + host = it[3]; + query = it[4]; + + if authstring then + auth = string.sub(authstring, 0, -2); + auth64 = ngx.encode_base64(auth); + ngx.var.authorization = "Basic " .. auth64; + end + + -- Default to / if none is set to avoid using the request_uri query + if not query then + query = "/"; + end + + return scheme .. host .. query; +'; + +add_header X-GG-Cache-Status $upstream_cache_status; +proxy_set_header Authorization $authorization; + +proxy_pass $proxyuri; +# Redirect back to ourselves on 301 replies +proxy_redirect ~^(.*)$ /cache/?uri=$1;