Files
infrastructure/roles/archweb/templates/nginx.d.conf.j2
Sven-Hendrik Haase dbdd000031 archweb: Ratelimit and cache mirrorlist endpoint
We saw a lot of requests and cache misses on this endpoint recently. Seems like a good candidate to cache and rate limit.
2026-09-09 23:59:02 +02:00

273 lines
7.6 KiB
Django/Jinja

# limit rss requests to 1 r/m
limit_req_zone $binary_remote_addr zone=rsslimit:8m rate=1r/m;
# limit mirrors/status/json requests to 1 r/m
limit_req_zone $binary_remote_addr zone=mirrorstatuslimit:8m rate=1r/m;
# limit mirrorlist requests to 1 r/m
limit_req_zone $binary_remote_addr zone=mirrorlistlimit:8m rate=1r/m;
# limit general requests to 5 r/s to block DoS attempts.
limit_req_zone $binary_remote_addr zone=archweblimit:10m rate=5r/s;
# limit http redirect
limit_req_zone $binary_remote_addr zone={{ http_redirect_ratelimit_zone }}:10m rate=20r/s;
limit_req_status 429;
uwsgi_cache_path /var/lib/nginx/cache levels=1:2 keys_zone=archwebcache:10m inactive=60m;
uwsgi_cache_key "$scheme$host$request_uri";
map $uri $cache_key {
default $scheme$host$request_uri;
/devel/mirrorauth/ $scheme$host$request_uri$http_authorization;
}
upstream archweb {
server unix:///run/uwsgi/archweb.sock;
}
{% if archweb_domains_templates -%}
{% for domain in archweb_domains_templates | dict2items(key_name='domain_name', value_name='template_name') %}
{% include domain['template_name'] %}
{% endfor %}
{%- endif %}
{% if archweb_domains_redirects %}
{% for domain in archweb_domains_redirects | dict2items(key_name='domain', value_name='redirect') %}
server {
include snippets/listen-80.conf;
server_name {{ domain['domain'] }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
include snippets/letsencrypt.conf;
location /.well-known/ {
include snippets/headers.conf;
add_header Access-Control-Allow-Origin *;
return 301 https://$server_name$request_uri;
}
location / {
limit_req zone=httpredirectlimit burst=10 nodelay;
# this try_files statement allows the limit_req to be enforced before the redirect.
try_files "" @https_redirect;
}
location @https_redirect {
access_log off;
return 301 https://$server_name$request_uri;
}
}
server {
include snippets/listen-443.conf;
server_name {{ domain['domain'] }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
acme_certificate letsencrypt;
ssl_certificate $acme_certificate;
ssl_certificate_key $acme_certificate_key;
location /.well-known/ {
include snippets/headers.conf;
add_header Access-Control-Allow-Origin *;
return 301 https://{{ archweb_domain }}{{ domain['redirect']|default('$request_uri') }};
}
location / {
access_log off;
return 301 https://{{ archweb_domain }}{{ domain['redirect']|default('$request_uri') }};
}
}
{% endfor %}
server {
{% else %}
server {
{% endif %}
include snippets/listen-80.conf;
server_name {{ archweb_domain }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
include snippets/letsencrypt.conf;
location / {
access_log off;
return 301 https://$server_name$request_uri;
}
}
server {
include snippets/listen-443.conf;
server_name {{ archweb_domain }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
acme_certificate letsencrypt;
ssl_certificate $acme_certificate;
ssl_certificate_key $acme_certificate_key;
location = /.well-known/matrix/server {
alias {{ archweb_dir }}/archlinux.org/.well-known/matrix/server;
default_type application/json;
}
location = /.well-known/matrix/client {
alias {{ archweb_dir }}/archlinux.org/.well-known/matrix/client;
default_type application/json;
include snippets/headers.conf;
add_header Access-Control-Allow-Origin *;
}
location = /robots.txt {
alias {{ archweb_dir }}/archlinux.org/robots.txt;
}
location = /humans.txt {
alias {{ archweb_dir }}/archlinux.org/humans.txt;
}
location = /google7827eadf026b4a87.html {
alias {{ archweb_dir }}/archlinux.org/google7827eadf026b4a87.html;
}
location = /BingSiteAuth.xml {
alias {{ archweb_dir }}/archlinux.org/BingSiteAuth.xml;
}
location = /favicon.ico {
alias {{ archweb_dir }}/collected_static/favicon.ico;
}
location ~ ^/pacman/(.*)$ {
return 301 https://pacman.archlinux.page/$1;
}
location /netcfg/ {
alias {{ archweb_dir }}/archlinux.org/netcfg/;
}
location /logos/ {
alias {{ archweb_dir }}/archlinux.org/logos/;
}
location /iso/ {
alias {{ archweb_rsync_iso_dir }};
location ~ ^/iso/.*\.(sig|torrent|txt)$ {
}
location /iso/ {
deny all;
}
}
# Cache django's css, js and png files.
location /static/ {
expires 30d;
include snippets/headers.conf;
add_header Pragma public;
add_header Cache-Control "public";
alias {{ archweb_dir }}/collected_static/;
}
location /img/ {
alias {{ archweb_dir }}/media/img/;
}
location /retro/ {
alias {{ archweb_retro_dir }};
}
# Rate limit all RSS feeds
location ~ (^/feeds/|\.xml$) {
include uwsgi_params;
uwsgi_pass archweb;
uwsgi_cache archwebcache;
uwsgi_cache_revalidate on;
include snippets/headers.conf;
add_header X-Cache-Status $upstream_cache_status;
limit_req zone=rsslimit burst=10 nodelay;
}
# Rate limit mirrorlist endpoint
location /mirrorlist {
include uwsgi_params;
uwsgi_pass archweb;
uwsgi_cache archwebcache;
uwsgi_cache_valid 5m;
uwsgi_cache_revalidate on;
uwsgi_cache_key $cache_key;
include snippets/headers.conf;
add_header X-Cache-Status $upstream_cache_status;
limit_req zone=mirrorlistlimit burst=10 nodelay;
}
# Rate limit mirror status json endpoint
location /mirrors/status/json {
include uwsgi_params;
uwsgi_pass archweb;
uwsgi_cache archwebcache;
uwsgi_cache_revalidate on;
uwsgi_cache_key $cache_key;
include snippets/headers.conf;
add_header X-Cache-Status $upstream_cache_status;
limit_req zone=mirrorstatuslimit burst=10 nodelay;
}
# Temporary redirects
location /people/trusted-user-fellows/ {
return 301 /people/package-maintainer-fellows/;
}
location /people/trusted-users/ {
return 301 /people/package-maintainers/;
}
location = /metrics {
if ($http_authorization != "Bearer {{ vault_archweb_metrics_token }}") {
return 403;
}
include uwsgi_params;
uwsgi_pass archweb;
}
location / {
access_log /var/log/nginx/{{ archweb_domain }}/access.log main;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_main;
include uwsgi_params;
uwsgi_pass archweb;
uwsgi_cache archwebcache;
uwsgi_cache_revalidate on;
uwsgi_cache_key $cache_key;
include snippets/headers.conf;
add_header X-Cache-Status $upstream_cache_status;
limit_req zone=archweblimit burst=10 nodelay;
}
}