mirror of
https://gitlab.archlinux.org/archlinux/infrastructure.git
synced 2026-09-11 14:10:51 +02:00
Leonidas SpyropoulosandLeonidas Spyropoulos
a37df93f21
Add complete codesearch role with zoekt implementation. Zoekt provides disk-based sharded indexing that avoids OOM issues on 8GB systems, with per-package .zoekt shard files and automatic discovery. - Implement fetch-sources script that clones archlinux/packaging/state to determine current package list and versions, then downloads PKGBUILDs and upstream sources from GitLab with parallel job support and idempotent re-runs - Add zoekt-webserver systemd service for search frontend - Add zoekt-index service/timer for incremental index generation - Configure nginx reverse proxy to port 6070 with TLS - Set up firewall rules for HTTP/HTTPS and monitoring access Architecture: - codesearch-extract.timer: Daily fetch trigger - codesearch-extract.service: Runs fetch-sources to download sources - codesearch-index.timer: Runs every 5 minutes after extraction - codesearch-index.service: Builds zoekt shards - codesearch-webserver.service: Serves search UI from disk shards Fixes: infrastructure#808 Signed-off-by: Leonidas Spyropoulos <artafinde@archlinux.org>
110 lines
3.3 KiB
YAML
110 lines
3.3 KiB
YAML
---
|
|
- name: Format codesearch data volume
|
|
filesystem:
|
|
fstype: btrfs
|
|
dev: "{{ codesearch_volume_device }}"
|
|
opts: "-L codesearch"
|
|
when: codesearch_volume_device is defined
|
|
|
|
- name: Mount codesearch data volume
|
|
mount:
|
|
path: /srv/codesearch
|
|
src: "{{ codesearch_volume_device }}"
|
|
fstype: btrfs
|
|
opts: defaults,compress=zstd
|
|
state: mounted
|
|
when: codesearch_volume_device is defined
|
|
|
|
- name: Create codesearch system user
|
|
user: name={{ codesearch_user }} shell=/sbin/nologin system=yes
|
|
|
|
- name: Create codesearch directories
|
|
file:
|
|
path: "{{ item }}"
|
|
state: directory
|
|
owner: "{{ codesearch_user }}"
|
|
group: "{{ codesearch_group }}"
|
|
mode: "0755"
|
|
loop:
|
|
- "{{ codesearch_sources_dir }}"
|
|
- "{{ codesearch_index_dir }}"
|
|
- "{{ codesearch_state_dir }}"
|
|
- "{{ codesearch_log_dir }}"
|
|
- "{{ codesearch_fetch_workdir }}"
|
|
- "{{ codesearch_state_repo_dir }}"
|
|
|
|
- name: Install logrotate config
|
|
copy: src=logrotate.conf dest=/etc/logrotate.d/codesearch owner=root group=root mode=0644
|
|
|
|
- name: Install required packages
|
|
pacman:
|
|
name:
|
|
- go
|
|
- parallel
|
|
state: present
|
|
|
|
# TODO: We should probably be installing zoekt from a package instead of building it ourselves, but this is easier for now.
|
|
- name: Install zoekt binaries
|
|
command: go install github.com/sourcegraph/zoekt/cmd/{{ item }}@{{ codesearch_zoekt_version }}
|
|
environment:
|
|
GOPATH: /usr/local
|
|
GOMODCACHE: /var/cache/go
|
|
loop:
|
|
- zoekt-index
|
|
- zoekt-webserver
|
|
changed_when: false
|
|
|
|
- name: Install source fetch script
|
|
template: src=fetch-sources.j2 dest=/usr/local/bin/fetch-sources owner=root group=root mode=0755
|
|
|
|
- name: Install codesearch index script
|
|
template: src=codesearch-index.sh.j2 dest=/usr/local/bin/codesearch-index owner=root group=root mode=0755
|
|
|
|
- name: Deploy systemd units
|
|
template: src={{ item }}.j2 dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
|
|
loop:
|
|
- codesearch-extract.service
|
|
- codesearch-extract.timer
|
|
- codesearch-index.service
|
|
- codesearch-index.timer
|
|
- codesearch-webserver.service
|
|
notify:
|
|
- Daemon reload
|
|
|
|
- name: Start and enable codesearch-webserver
|
|
systemd: name=codesearch-webserver state=started enabled=yes daemon_reload=yes
|
|
|
|
- name: Start and enable codesearch timers
|
|
systemd:
|
|
name: "{{ item }}"
|
|
state: started
|
|
enabled: true
|
|
loop:
|
|
- codesearch-extract.timer
|
|
- codesearch-index.timer
|
|
|
|
- name: Make nginx log dir
|
|
file: path=/var/log/nginx/{{ codesearch_domain }} state=directory owner=root group=root mode=0755
|
|
|
|
- name: Set up nginx
|
|
template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/codesearch.conf owner=root group=root mode=0644
|
|
notify:
|
|
- Reload nginx
|
|
tags: ['nginx']
|
|
|
|
- name: Open port 80 for codesearch (HTTP redirect + ACME)
|
|
ansible.posix.firewalld: service=http state=enabled permanent=true immediate=yes
|
|
tags:
|
|
- firewall
|
|
|
|
- name: Open port 443 for codesearch (HTTPS)
|
|
ansible.posix.firewalld: service=https state=enabled permanent=true immediate=yes
|
|
tags:
|
|
- firewall
|
|
|
|
- name: Allow prometheus exporter access from monitoring.archlinux.org
|
|
ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
|
|
rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=9100 accept"
|
|
tags:
|
|
- firewall
|