Merge branch 'introduce_backups'

roles/backups/files/backup_script.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+# Inspiration of script from: https://borgbackup.readthedocs.io/en/stable/quickstart.html#automating-backups
+
+logInfo() { printf '%(%Y-%m-%d %H:%m:%S)T [INFO]: %s\n' -1 "$*" >&2; }
+
+configurationFileLocation=$1
+
+# Working on the bulk directory which is mounted on a very big SSD, this way we don't have to wory about
+# running out of disk space. Note however that borg specifically might make use of cache directories
+# in the home directory which aren't on this disk, let's tackle this problem when it actually starts failing...
+workDirPath=/bulk/backup_work_dir
+logInfo "Creating working directory at: $workDirPath..."
+mkdir -p "$workDirPath"
+
+cleanup() {
+    logInfo "Removing working directory at: $workDirPath..."
+    rm -rf "$workDirPath"
+}
+trap 'logInfo "Backup interrupted"; cleanup; exit 2' INT TERM
+trap 'logInfo "Backup completed, cleaning up..."; cleanup' EXIT
+
+# region: ------ DB_BACKUPS -----------------------------------------------------------------------
+createDatabaseBackup() {
+    host=$1
+    dbname=$2
+    username=$3
+    password=$4
+    targetFolderPath=$5
+
+    logInfo "Dumping database: $dbname to $targetFolderPath..."
+
+    # Getting the correct version tools installed on the host proofed to be a very frustrating experience.
+    # So instead we'll do the dumping on the container.
+    postgresContainerName='postgres-postgres-1'
+    containerDumpPath=/dump.sql
+
+    logInfo "Dumping database $dbname on container: $postgresContainerName..."
+    docker exec "$postgresContainerName" bash -c "(export PGPASSWORD='$password'; pg_dump $dbname \
+        --file $containerDumpPath \
+        --host $host \
+        --username $username)"
+
+    logInfo "Extracting the archive from the container..."
+    docker cp "$postgresContainerName:$containerDumpPath" "$targetFolderPath/$dbname.sql" 2>/dev/null
+
+    logInfo "Removing the file from the docker container..."
+    docker exec "$postgresContainerName" rm "$containerDumpPath"
+}
+
+createAllDatabaseBackups() {
+    nrOfConfigurations="$(yq '.database_backups | length' <"$configurationFileLocation")"
+    logInfo "Backing up from $nrOfConfigurations database configurations..."
+
+    postgresBackupDirectory="$workDirPath/postgres"
+    mkdir -p "$postgresBackupDirectory"
+
+    for ((i = 0 ; i < "$nrOfConfigurations" ; i++)); do
+        dbConfiguration="$(yq ".database_backups[$i]" <"$configurationFileLocation")"
+        host="$(echo "$dbConfiguration" | jq -r '.host')"
+        dbname="$(echo "$dbConfiguration" | jq -r '.dbname')"
+        username="$(echo "$dbConfiguration" | jq -r '.username')"
+        password="$(echo "$dbConfiguration" | jq -r '.password')"
+        targetFolderPath="$postgresBackupDirectory"
+
+        createDatabaseBackup "$host" "$dbname" "$username" "$password" "$targetFolderPath"
+    done
+}
+# endregion: --- DB_BACKUPS -----------------------------------------------------------------------
+
+# region: ------ DOCKER_VOLUME_BACKUPS ------------------------------------------------------------
+createDockerVolumeBackup() {
+    containerName=$1
+    volumeName=$2
+    specificVolumeBackupPath=$3
+
+    logInfo "Backup up Docker volume: $volumeName from running container: $containerName..."
+
+    logInfo "Stopping container: $containerName..."
+    docker stop "$containerName"
+
+    logInfo "Starting new container which copies over files..."
+    start=$SECONDS
+    docker run --rm -v "$volumeName:/volume" -v "$specificVolumeBackupPath:/target" --entrypoint "ash"\
+        alpine -c "cp -rf /volume/* /target/"
+
+    elapsedSeconds=$(( SECONDS - start ))
+    logInfo "Copying succeeded (in $elapsedSeconds seconds), restarting container..."
+    docker start "$containerName"
+}
+
+createAllDockerVolumeBackups() {
+    nrOfConfigurations="$(yq '.docker_volume_backups | length' <"$configurationFileLocation")"
+    logInfo "Backing up from $nrOfConfigurations docker configurations..."
+
+    dockerVolumeBackupPath="$workDirPath/docker_volumes"
+    mkdir -p "$dockerVolumeBackupPath"
+
+    for ((i = 0 ; i < "$nrOfConfigurations" ; i++)); do
+        dockerConfiguration="$(yq ".docker_volume_backups[$i]" <"$configurationFileLocation")"
+        containerName="$(echo "$dockerConfiguration" | jq -r '.container_name')"
+        volumeName="$(echo "$dockerConfiguration" | jq -r '.volume_name')"
+
+        specificVolumeBackupPath="$dockerVolumeBackupPath/$volumeName"
+        mkdir -p "$specificVolumeBackupPath"
+
+        createDockerVolumeBackup "$containerName" "$volumeName" "$specificVolumeBackupPath"
+    done
+}
+# endregion: --- DOCKER_VOLUME_BACKUPS ------------------------------------------------------------
+
+# region: ------ BORG_BACKUPS ---------------------------------------------------------------------
+createArchiveInRepository() {
+    logInfo "Creating new archive in Borg repository (at $BORG_REPO)..."
+
+    (
+        cd "$workDirPath"
+
+        # Note that both BORG_PASSPHRASE and BORG_REPO should be set, otherwise a password prompt will be present...
+        borg create --stats --verbose --show-rc --compression zstd,11 \
+            "::{fqdn}-{now:%Y-%m-%d}" \
+            ./docker_volumes ./postgres
+
+        logInfo "Pruning old backups..."
+        # Copied from: https://borgbackup.readthedocs.io/en/stable/quickstart.html as it seems
+        # like good defaults.
+        borg prune --verbose --glob-archives '{fqdn}-*' --show-rc \
+            --keep-daily 7 --keep-weekly 4 --keep-monthly 6
+
+        logInfo "Compacting repository..."
+        borg compact --verbose --show-rc
+    )
+}
+# endregion: --- BORG_BACKUPS ---------------------------------------------------------------------
+
+# region: ------ PREPARE_BACKUP_FILES -------------------------------------------------------------
+createAllDatabaseBackups
+echo
+createAllDockerVolumeBackups
+echo
+createArchiveInRepository
+# endregion: --- PREPARE_BACKUP_FILES -------------------------------------------------------------

roles/backups/files/borg_backup.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=BorgBase backup timer
+
+[Timer]
+OnCalendar=daily
+RandomizedDelaySec=900
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/backups/tasks/main.yml

@@ -0,0 +1,60 @@
+---
+# From within the script we're pushing backups to a specialised service (BorgBackup), This step ensure that an SSH key is present to use
+# for verification on that service. Currently it has to be manually read out and entered in the service. This step has to be repeated
+# when freshly applying this setup.
+- name: Generate an OpenSSH keypair with the default values (4096 bits, rsa)
+  become: true
+  community.crypto.openssh_keypair:
+    path: "{{ backup_script_ssh_key_location }}"
+# Needed for the task after this apparently...
+- name: Install SSH config file
+  become: true
+  ansible.builtin.template:
+    src: ssh_config
+    dest: /root/.ssh/config
+    owner: root
+    group: root
+    mode: '0700'
+- name: Copy over script
+  become: true
+  ansible.builtin.copy:
+    src: backup_script.sh
+    dest: "{{ backups_script_path }}"
+    owner: root
+    group: root
+    mode: '0700'
+- name: Ensure directory for configuration file exists
+  become: true
+  ansible.builtin.file:
+    path: "{{ backups_configuration_path | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+- name: Copy over configuration
+  become: true
+  ansible.builtin.template:
+    src: backup_configuration.yaml
+    dest: "{{ backups_configuration_path }}"
+    owner: root
+    group: root
+    mode: '0400'
+- name: Install BorgBase backup service file
+  become: true
+  ansible.builtin.template:
+    src: borg_backup.service.j2
+    dest: "/lib/systemd/system/borg_backup.service"
+    mode: '0644'
+- name: Install BorgBase backup timer file
+  become: true
+  ansible.builtin.copy:
+    src: borg_backup.timer
+    dest: "/lib/systemd/system/borg_backup.timer"
+    mode: '0644'
+- name: Enable the newly added systemd timer
+  become: true
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+    name: "borg_backup.timer"
+    state: started
+    enabled: true

roles/backups/templates/backup_configuration.yaml

@@ -0,0 +1,19 @@
+---
+database_backups:
+  - host: postgres.kleinendorst.info
+    dbname: wedding
+    username: wedding
+    password: "{{ wedding_postgres_pass }}"
+docker_volume_backups:
+  - volume_name: actual_data
+    container_name: actual-server
+  - volume_name: grafana_data
+    container_name: grafana-server
+  - volume_name: changedetection_changedetection_data
+    container_name: changedetection-changedetection-server-1
+  - volume_name: hoarder_hoarder_data
+    container_name: hoarder-web-1
+  - volume_name: hoarder_meilisearch
+    container_name: hoarder-meilisearch-1
+  - volume_name: portainer_data
+    container_name: portainer

roles/backups/templates/borg_backup.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=BorgBase backup service
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+ExecStart={{ backups_script_path }} {{ backups_configuration_path }}
+Environment="BORG_REPO={{ borg_base.repo_url }}"
+Environment="BORG_PASSPHRASE={{ borg_backup_password }}"
+
+[Install]
+WantedBy=default.target

roles/backups/templates/ssh_config

@@ -0,0 +1,5 @@
+Host {{ borg_base.remote_host }}
+    HostName {{ borg_base.remote_host }}
+    User {{ borg_base.remote_user }}
+    IdentityFile {{ backup_script_ssh_key_location }}
+    StrictHostKeyChecking accept-new

roles/backups/vars/main/defaults.yml

@@ -0,0 +1,4 @@
+---
+backup_script_ssh_key_location: /root/.ssh/id_ssh_rsa
+backups_script_path: /usr/local/bin/backup_script.sh
+backups_configuration_path: /etc/borg_backup_script/backup_configuration.yaml

roles/backups/vars/main/vault.yml

@@ -0,0 +1,18 @@
+$ANSIBLE_VAULT;1.1;AES256
+36663737323462306164663362663633363838633165303464666233333364323330613933326237
+3565663233613037336633313537346534333432633036360a353064366464333164393161653038
+30633966613031363932633736333337653464373866333836353032356431393866303836343166
+6464333031323639660a306631363234383366643435366536323861356434393566643633643839
+35313064653536393366366536386331663062663132313331353238653933356234333338343436
+32616565323636633239346366323934303766353936653336353063373663623932353532386532
+32633736323866313133363438373639396663333737363536353731353236303333626364386632
+64363336356566653130303765396232646231333436366434353634316631313365373561383636
+38386636623265643762613065376362653964653935306338653763306137323165346332623264
+33636164613562636164363065623564363965626235643238363630666639363866663631643530
+65613938663131396630303565646335623764353830356536376465346339363034316666306134
+31353731316430663136613061386566613832626234656337343065363331636239326365343762
+33663965626538643937323832663638613766323331623133376632666131353936346238386437
+61306135386131653466633331313165626162306639323633383133643761633466373234353134
+39323237666334323232623230643734363765376163333762643962356365343364383939333132
+63363961383934643935323264326133313135336638323833336539393136306435663134333930
+32343762623636323637383530366434326537313431636131343533613733613063

roles/packages/tasks/main.yml

@@ -3,11 +3,15 @@
   become: true
   ansible.builtin.apt:
     pkg:
+      - tldr
+      - tree
       - git
       - vim
       - dnsutils
       - rsyslog
       - snapd
+      - yq
+      - borgbackup
     state: present
 - name: Install Snap Core
   become: true