infra/nix/modules/wanderllama/borgmatic.nix

{
  pkgs,
  lib,
  options,
  config,
  ...
}:
{
  options =
    with lib;
    with types;
    {
      wanderllama.borgmatic.enable = mkOption {
        default = false;
        type = bool;
        description = ''
          Enable borgmatic/borgbase support.
        '';
      };
      wanderllama.borgmatic.jobName = mkOption {
        default = "";
        type = str;
        description = ''
          Name of the backup job.
        '';
      };
      wanderllama.borgmatic.username = mkOption {
        default = "";
        type = nullOr str;
        description = ''
          Username for borgbase repository.
        '';
      };
      wanderllama.borgmatic.hostname = mkOption {
        default = "";
        type = nullOr str;
        description = ''
          Hostname for borgbase repository.
        '';
      };
      #wanderllama.borgmatic.username = mkOption {
      #  default = "";
      #  type = nullOr str;
      #  description = ''
      #    Username for borgbase repository.
      #  '';
      #};
      wanderllama.borgmatic.retention.hours = mkOption {
        default = 12;
        type = number;
        description = ''
          How many hours to keep the 0th-tiered backups.
        '';
      };
      wanderllama.borgmatic.retention.days = mkOption {
        default = 7;
        type = number;
        description = ''
          How many days to keep the 1st-tiered/"child" backups.
        '';
      };
      wanderllama.borgmatic.retention.weeks = mkOption {
        default = 4;
        type = number;
        description = ''
          How many weeks to keep the 2nd-tiered/"father" backups.
        '';
      };
      wanderllama.borgmatic.retention.months = mkOption {
        default = 3;
        type = number;
        description = ''
          How many days to keep the 3rd-tiered/"grandfather" backups.
        '';
      };
      wanderllama.borgmatic.directories = mkOption {
        default = [
          "/home"
          "/root"
        ];
        type = listOf str;
        description = ''
          Directories to back up.
        '';
      };
      wanderllama.borgmatic.excludes = mkOption {
        default = [ ];
        type = listOf str;
        description = ''
          Glob patterns to exclude from backup.
        '';
      };
      wanderllama.borgmatic.calendar = mkOption {
        default = [ "*-*-* 04:00:00" ];
        type = listOf str;
        description = ''
          The OnCalendar value for the backup timer.
        '';
      };
      wanderllama.borgmatic.ssh = mkOption {
        default = "";
        type = str;
        description = ''
          ssh -i path/to/key
        '';
      };
      wanderllama.borgmatic.passcmd = mkOption {
        default = "";
        type = str;
        description = ''
          cat /path/to/passphrase
        '';
      };
      wanderllama.borgmatic.label = mkOption {
        default = "";
        type = str;
        description = ''
          short label
        '';
      };
      wanderllama.borgmatic.appendOnly = mkOption {
        default = true;
        type = bool;
        description = ''
          append-only or not
        '';
      };
    };

  config =
    with lib;
    let
      cfg = config.wanderllama.borgmatic;
      # N.B. this should generate valid YAML because YAML is 99.999% a superset of JSON.
      source_directories = concatStringsSep ", " (map builtins.toJSON cfg.directories);
      exclude_patterns = concatStringsSep ", " (map builtins.toJSON cfg.excludes);
    in
    {
      #''systemd.services."generate-borgmatic-secrets" = {
      #''  enable = cfg.enable;
      #''  description = "Generate secrets used by borgmatic";
      #''  serviceConfig = {
      #''    Type = "oneshot";
      #''    UMask = 0077;
      #''  };
      #''  script = ''
      #''    mkdir -p /secrets/borgmatic/borgbase
      #''    if [[ ! -f /secrets/borgmatic/borgbase/ssh ]]; then
      #''      ssh-keygen -t ed25519 -q -f /secrets/borgmatic/borgbase/ssh
      #''    fi
      #''    if [[ ! -f /secrets/borgmatic/borgbase/passphrase ]]; then
      #''      { tr -dc A-Za-z0-9 </dev/urandom | head -c 80; echo; } > \
      #''        /secrets/borgmatic/borgbase/passphrase
      #''    fi
      #''  '';
      #''  path = [ pkgs.openssh ];
      #''  wantedBy = [
      #''    "multi-user.target"
      #''    "backup.service"
      #''  ];
      #''};

      systemd.services.backup = {
        enable = cfg.enable;
        serviceConfig = {
          Type = "oneshot";
          Nice = 19;
          CPUSchedulingPolicy = "batch";
          IOSchedulingClass = "best-effort";
          IOSchedulingPriority = 7;
          IOWeight = 100;
          PrivateTmp = true;
          Restart = "on-failure";
          RestartSec = 30;
        };
        unitConfig.Wants = [ "network-online.target" ];
        path = [ pkgs.borgmatic ];
        script = ''
          borgmatic -c /etc/borgmatic/borgbase.yaml \
            --syslog-verbosity -1 \
            --verbosity 1
        '';
      };

      systemd.timers.backup = {
        enable = cfg.enable;
        timerConfig = {
          OnCalendar = cfg.calendar;
          OnActiveSec = 600;
        };
        wantedBy = [ "timers.target" ];
      };

      environment.etc."borgmatic/borgbase.yaml".text = ''
        # location:
        source_directories: [${source_directories}]
        exclude_patterns: [${exclude_patterns}]
        repositories:
          - path: ssh://${cfg.username}@${cfg.hostname}/./repo
            label: "${cfg.label}"
            append_only: true
        # one_file_system: true
        #retention:
        keep_hourly: ${toString cfg.retention.hours}
        keep_daily: ${toString cfg.retention.days}
        keep_weekly: ${toString cfg.retention.weeks}
        keep_monthly: ${toString cfg.retention.months}
        #consistency:
        #  checks:
        #    - repository
        #    - archives
        skip_actions: ["check"]
        check_last: 5

        retries: 3
        retry_wait: 5

        compression: auto,zstd
        encryption_passcommand: ${toString cfg.passcmd}
        ssh_command: ${toString cfg.ssh}

        progress: true
        statistics: true

        loki:
          # Grafana loki log URL to notify when a backup begins,
          # ends, or fails.
          url: https://${domain}/loki/api/v1/push

          # Allows setting custom labels for the logging stream. At
          # least one label is required. "__hostname" gets replaced by
          # the machine hostname automatically. "__config" gets replaced
          # by the name of the configuration file. "__config_path" gets
          # replaced by the full path of the configuration file.
          labels:
              app: borgmatic
              config: __config
              host: __hostname
              nodename: __hostname
      '';

      # Kind of perplexed this is such an involved process to safely retrieve ssh
      # host keys.  Sure you can just "ssh yourhost", visually verify, then hit
      # yes, writing some random file.  Generally you never want this on a system
      # with a declarative configuration (i.e. any system with something like
      # ansible or nix).  OpenSSH has come a long way but sometimes I can't help
      # but wonder how big of a rock do they live under?
      #
      # 1. Refer to https://www.borgbase.com/setup see the ssh fingerprints
      # 2. ssh-keyscan yourhost.repo.borgbase.com > /tmp/scan 2> /dev/null
      # 3. ssh-keygen -l -f /tmp/scan
      # 4. Verify the fingerprints match
      # 5. Then copy-paste the text after the host name below for each key type.

      programs.ssh.knownHosts."borgbase-nx/rsa" = {
        hostNames = [ cfg.hostname ];
        publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwHsO5g7kAEpqcK4bpHCUKYV1cKCUNwVEVsDQyfj7N8L92E21n+aEhIX2Nh/kFs1W9D/pgsWQBAbco9e/ORuagHrO8hUQtbda5Z31PAo4eipwP17VQr5rF3seaJJNFV72v89PGwMOWQwvoJte+yngC6PYGKJ+w63SRtflihAmf4xa5Tci/f6jbX6t32m2F3bnephVzQO6anGXvGPR8QYQXzSu/27+LaKnLd2Kugb1Ytbo0+6kioa60HWejIZ/mCrCHXYpi0jAllaYEuAsTqFWf/OFUHrKWwRAJD0TV43O1++vLlxY85oQxIgc4oUbm93dXmDBssrTnqqq2jqonteUr";
      };
      programs.ssh.knownHosts."borgbase-nx/ecdsa" = {
        hostNames = [ cfg.hostname ];
        publicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOstKfBbwVOYQh3J7X4nzd6/VYgLfaucP9z5n4cpSzcZAOKGh6jH8e1mhQ4YupthlsdPKyFFZ3pKo4mTaRRuiJo=";
      };
      programs.ssh.knownHosts."borgbase-nx/ed25519" = {
        hostNames = [ cfg.hostname ];
        publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMS3185JdDy7ffnr0nLWqVy8FaAQeVh1QYUSiNpW5ESq";
      };
    };
}