docs/ci: Document setting up the http cache for traces.
Reviewed-by: Daniel Stone <daniel@fooishbar.org> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8727>
This commit is contained in:
parent
04a2f1462e
commit
2e2edaa89b
|
@ -128,3 +128,41 @@ that gets used to start a nginx server.
|
|||
|
||||
Once you've updated your runners' configs, restart with ``sudo service
|
||||
gitlab-runner restart``
|
||||
|
||||
Caching downloads
|
||||
-----------------
|
||||
|
||||
To improve the runtime for downloading traces during traces job runs, you will
|
||||
want a pass-through HTTP cache. On your runner box, install nginx:
|
||||
|
||||
.. code-block:: console
|
||||
sudo apt install nginx libnginx-mod-http-lua
|
||||
|
||||
Add the server setup files:
|
||||
|
||||
.. literalinclude: fdo-cache:
|
||||
:name: /etc/nginx/sites-available/fdo-cache
|
||||
|
||||
.. literalinclude: uri-caching.conf:
|
||||
:name: /etc/nginx/sites-available/snippets/uri-caching.conf
|
||||
|
||||
Edit the listener addresses in fdo-cache to suit the ethernet interface that
|
||||
your devices are on.
|
||||
|
||||
Enable the site and restart nginx:
|
||||
|
||||
.. code-block:: console
|
||||
sudo ln -s /etc/nginx/sites-available/fdo-cache /etc/nginx/sites-enabled/fdo-cache
|
||||
sudo service nginx restart
|
||||
|
||||
# First download will hit the internet
|
||||
wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
|
||||
# Second download should be cached.
|
||||
wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
|
||||
|
||||
Now, set ``download-url`` in your ``traces-*.yml`` entry to something like
|
||||
``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public``
|
||||
and you should have cached downloads for traces. Add it to
|
||||
``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you
|
||||
can use it for cached artifact downloads instead of going all the way to
|
||||
freedesktop.org on each job.
|
||||
|
|
|
@ -0,0 +1,44 @@
|
|||
proxy_cache_path /var/cache/nginx/ levels=1:2 keys_zone=my_cache:10m max_size=24g inactive=48h use_temp_path=off;
|
||||
|
||||
server {
|
||||
listen 10.42.0.1:8888 default_server;
|
||||
listen 127.0.0.1:8888 default_server;
|
||||
listen [::]:8888 default_server;
|
||||
resolver 8.8.8.8;
|
||||
|
||||
root /var/www/html;
|
||||
|
||||
# Add index.php to the list if you are using PHP
|
||||
index index.html index.htm index.nginx-debian.html;
|
||||
|
||||
server_name _;
|
||||
|
||||
add_header X-GG-Cache-Status $upstream_cache_status;
|
||||
proxy_cache my_cache;
|
||||
|
||||
location /cache_gitlab_artifacts {
|
||||
internal;
|
||||
# Gitlabs http server puts everything as no-cache even though
|
||||
# the artifacts URLS don't change. So enforce a long validity
|
||||
# time and ignore the headers that defeat caching
|
||||
proxy_cache_valid 200 48h;
|
||||
proxy_ignore_headers Cache-Control Set-Cookie;
|
||||
include snippets/uri-caching.conf;
|
||||
}
|
||||
|
||||
location /cache {
|
||||
# special case gitlab artifacts
|
||||
if ($arg_uri ~* /.*gitlab.*artifacts(\/|%2F)raw/ ) {
|
||||
rewrite ^ /cache_gitlab_artifacts;
|
||||
}
|
||||
# Set a really low validity together with cache revalidation; Our goal
|
||||
# for caching isn't to lower the number of http requests but to
|
||||
# lower the amount of data transfer. Also for some test
|
||||
# scenarios (typical manual tests) the file at a given url
|
||||
# might get modified so avoid confusion by ensuring
|
||||
# revalidations happens often.
|
||||
proxy_cache_valid 200 10s;
|
||||
proxy_cache_revalidate on;
|
||||
include snippets/uri-caching.conf;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
set $authorization '';
|
||||
|
||||
set_by_lua $proxyuri '
|
||||
unescaped = ngx.unescape_uri(ngx.var.arg_uri);
|
||||
it, err = ngx.re.match(unescaped, "(https?://)(.*@)?([^/]*)(/.*)?");
|
||||
if not it then
|
||||
-- Hack to cause nginx to return 404
|
||||
return "http://localhost/404"
|
||||
end
|
||||
|
||||
scheme = it[1];
|
||||
authstring = it[2];
|
||||
host = it[3];
|
||||
query = it[4];
|
||||
|
||||
if authstring then
|
||||
auth = string.sub(authstring, 0, -2);
|
||||
auth64 = ngx.encode_base64(auth);
|
||||
ngx.var.authorization = "Basic " .. auth64;
|
||||
end
|
||||
|
||||
-- Default to / if none is set to avoid using the request_uri query
|
||||
if not query then
|
||||
query = "/";
|
||||
end
|
||||
|
||||
return scheme .. host .. query;
|
||||
';
|
||||
|
||||
add_header X-GG-Cache-Status $upstream_cache_status;
|
||||
proxy_set_header Authorization $authorization;
|
||||
|
||||
proxy_pass $proxyuri;
|
||||
# Redirect back to ourselves on 301 replies
|
||||
proxy_redirect ~^(.*)$ /cache/?uri=$1;
|
Loading…
Reference in New Issue