diff --git a/roles/backups/files/backup_script.sh b/roles/backups/files/backup_script.sh new file mode 100644 index 0000000..ad22196 --- /dev/null +++ b/roles/backups/files/backup_script.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +# Inspiration of script from: https://borgbackup.readthedocs.io/en/stable/quickstart.html#automating-backups + +logInfo() { printf '%(%Y-%m-%d %H:%m:%S)T [INFO]: %s\n' -1 "$*" >&2; } + +configurationFileLocation=$1 + +# Working on the bulk directory which is mounted on a very big SSD, this way we don't have to wory about +# running out of disk space. Note however that borg specifically might make use of cache directories +# in the home directory which aren't on this disk, let's tackle this problem when it actually starts failing... +workDirPath=/bulk/backup_work_dir +logInfo "Creating working directory at: $workDirPath..." +mkdir -p "$workDirPath" + +cleanup() { + logInfo "Removing working directory at: $workDirPath..." + rm -rf "$workDirPath" +} +trap 'logInfo "Backup interrupted"; cleanup; exit 2' INT TERM +trap 'logInfo "Backup completed, cleaning up..."; cleanup' EXIT + +# region: ------ DB_BACKUPS ----------------------------------------------------------------------- +createDatabaseBackup() { + host=$1 + dbname=$2 + username=$3 + password=$4 + targetFolderPath=$5 + + logInfo "Dumping database: $dbname to $targetFolderPath..." + + # Getting the correct version tools installed on the host proofed to be a very frustrating experience. + # So instead we'll do the dumping on the container. + postgresContainerName='postgres-postgres-1' + containerDumpPath=/dump.sql + + logInfo "Dumping database $dbname on container: $postgresContainerName..." + docker exec "$postgresContainerName" bash -c "(export PGPASSWORD='$password'; pg_dump $dbname \ + --file $containerDumpPath \ + --host $host \ + --username $username)" + + logInfo "Extracting the archive from the container..." + docker cp "$postgresContainerName:$containerDumpPath" "$targetFolderPath/$dbname.sql" 2>/dev/null + + logInfo "Removing the file from the docker container..." + docker exec "$postgresContainerName" rm "$containerDumpPath" +} + +createAllDatabaseBackups() { + nrOfConfigurations="$(yq '.database_backups | length' <"$configurationFileLocation")" + logInfo "Backing up from $nrOfConfigurations database configurations..." + + postgresBackupDirectory="$workDirPath/postgres" + mkdir -p "$postgresBackupDirectory" + + for ((i = 0 ; i < "$nrOfConfigurations" ; i++)); do + dbConfiguration="$(yq ".database_backups[$i]" <"$configurationFileLocation")" + host="$(echo "$dbConfiguration" | jq -r '.host')" + dbname="$(echo "$dbConfiguration" | jq -r '.dbname')" + username="$(echo "$dbConfiguration" | jq -r '.username')" + password="$(echo "$dbConfiguration" | jq -r '.password')" + targetFolderPath="$postgresBackupDirectory" + + createDatabaseBackup "$host" "$dbname" "$username" "$password" "$targetFolderPath" + done +} +# endregion: --- DB_BACKUPS ----------------------------------------------------------------------- + +# region: ------ DOCKER_VOLUME_BACKUPS ------------------------------------------------------------ +createDockerVolumeBackup() { + containerName=$1 + volumeName=$2 + specificVolumeBackupPath=$3 + + logInfo "Backup up Docker volume: $volumeName from running container: $containerName..." + + logInfo "Stopping container: $containerName..." + docker stop "$containerName" + + logInfo "Starting new container which copies over files..." + start=$SECONDS + docker run --rm -v "$volumeName:/volume" -v "$specificVolumeBackupPath:/target" --entrypoint "ash"\ + alpine -c "cp -rf /volume/* /target/" + + elapsedSeconds=$(( SECONDS - start )) + logInfo "Copying succeeded (in $elapsedSeconds seconds), restarting container..." + docker start "$containerName" +} + +createAllDockerVolumeBackups() { + nrOfConfigurations="$(yq '.docker_volume_backups | length' <"$configurationFileLocation")" + logInfo "Backing up from $nrOfConfigurations docker configurations..." + + dockerVolumeBackupPath="$workDirPath/docker_volumes" + mkdir -p "$dockerVolumeBackupPath" + + for ((i = 0 ; i < "$nrOfConfigurations" ; i++)); do + dockerConfiguration="$(yq ".docker_volume_backups[$i]" <"$configurationFileLocation")" + containerName="$(echo "$dockerConfiguration" | jq -r '.container_name')" + volumeName="$(echo "$dockerConfiguration" | jq -r '.volume_name')" + + specificVolumeBackupPath="$dockerVolumeBackupPath/$volumeName" + mkdir -p "$specificVolumeBackupPath" + + createDockerVolumeBackup "$containerName" "$volumeName" "$specificVolumeBackupPath" + done +} +# endregion: --- DOCKER_VOLUME_BACKUPS ------------------------------------------------------------ + +# region: ------ BORG_BACKUPS --------------------------------------------------------------------- +createArchiveInRepository() { + logInfo "Creating new archive in Borg repository (at $BORG_REPO)..." + + ( + cd "$workDirPath" + + # Note that both BORG_PASSPHRASE and BORG_REPO should be set, otherwise a password prompt will be present... + borg create --stats --verbose --show-rc --compression zstd,11 \ + "::{fqdn}-{now:%Y-%m-%d}" \ + ./docker_volumes ./postgres + + logInfo "Pruning old backups..." + # Copied from: https://borgbackup.readthedocs.io/en/stable/quickstart.html as it seems + # like good defaults. + borg prune --verbose --glob-archives '{fqdn}-*' --show-rc \ + --keep-daily 7 --keep-weekly 4 --keep-monthly 6 + + logInfo "Compacting repository..." + borg compact --verbose --show-rc + ) +} +# endregion: --- BORG_BACKUPS --------------------------------------------------------------------- + +# region: ------ PREPARE_BACKUP_FILES ------------------------------------------------------------- +createAllDatabaseBackups +echo +createAllDockerVolumeBackups +echo +createArchiveInRepository +# endregion: --- PREPARE_BACKUP_FILES ------------------------------------------------------------- diff --git a/roles/backups/files/borg_backup.timer b/roles/backups/files/borg_backup.timer new file mode 100644 index 0000000..6ee4d8c --- /dev/null +++ b/roles/backups/files/borg_backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=BorgBase backup timer + +[Timer] +OnCalendar=daily +RandomizedDelaySec=900 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/roles/backups/tasks/main.yml b/roles/backups/tasks/main.yml new file mode 100644 index 0000000..3a5a5d4 --- /dev/null +++ b/roles/backups/tasks/main.yml @@ -0,0 +1,60 @@ +--- +# From within the script we're pushing backups to a specialised service (BorgBackup), This step ensure that an SSH key is present to use +# for verification on that service. Currently it has to be manually read out and entered in the service. This step has to be repeated +# when freshly applying this setup. +- name: Generate an OpenSSH keypair with the default values (4096 bits, rsa) + become: true + community.crypto.openssh_keypair: + path: "{{ backup_script_ssh_key_location }}" +# Needed for the task after this apparently... +- name: Install SSH config file + become: true + ansible.builtin.template: + src: ssh_config + dest: /root/.ssh/config + owner: root + group: root + mode: '0700' +- name: Copy over script + become: true + ansible.builtin.copy: + src: backup_script.sh + dest: "{{ backups_script_path }}" + owner: root + group: root + mode: '0700' +- name: Ensure directory for configuration file exists + become: true + ansible.builtin.file: + path: "{{ backups_configuration_path | dirname }}" + state: directory + owner: root + group: root + mode: '0755' +- name: Copy over configuration + become: true + ansible.builtin.template: + src: backup_configuration.yaml + dest: "{{ backups_configuration_path }}" + owner: root + group: root + mode: '0400' +- name: Install BorgBase backup service file + become: true + ansible.builtin.template: + src: borg_backup.service.j2 + dest: "/lib/systemd/system/borg_backup.service" + mode: '0644' +- name: Install BorgBase backup timer file + become: true + ansible.builtin.copy: + src: borg_backup.timer + dest: "/lib/systemd/system/borg_backup.timer" + mode: '0644' +- name: Enable the newly added systemd timer + become: true + ansible.builtin.systemd_service: + daemon_reload: true + name: "borg_backup.timer" + state: started + enabled: true diff --git a/roles/backups/templates/backup_configuration.yaml b/roles/backups/templates/backup_configuration.yaml new file mode 100644 index 0000000..2a5f4a6 --- /dev/null +++ b/roles/backups/templates/backup_configuration.yaml @@ -0,0 +1,19 @@ +--- +database_backups: + - host: postgres.kleinendorst.info + dbname: wedding + username: wedding + password: "{{ wedding_postgres_pass }}" +docker_volume_backups: + - volume_name: actual_data + container_name: actual-server + - volume_name: grafana_data + container_name: grafana-server + - volume_name: changedetection_changedetection_data + container_name: changedetection-changedetection-server-1 + - volume_name: hoarder_hoarder_data + container_name: hoarder-web-1 + - volume_name: hoarder_meilisearch + container_name: hoarder-meilisearch-1 + - volume_name: portainer_data + container_name: portainer diff --git a/roles/backups/templates/borg_backup.service.j2 b/roles/backups/templates/borg_backup.service.j2 new file mode 100644 index 0000000..b8a6793 --- /dev/null +++ b/roles/backups/templates/borg_backup.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=BorgBase backup service +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=root +Group=root +ExecStart={{ backups_script_path }} {{ backups_configuration_path }} +Environment="BORG_REPO={{ borg_base.repo_url }}" +Environment="BORG_PASSPHRASE={{ borg_backup_password }}" + +[Install] +WantedBy=default.target diff --git a/roles/backups/templates/ssh_config b/roles/backups/templates/ssh_config new file mode 100644 index 0000000..8a8252e --- /dev/null +++ b/roles/backups/templates/ssh_config @@ -0,0 +1,5 @@ +Host {{ borg_base.remote_host }} + HostName {{ borg_base.remote_host }} + User {{ borg_base.remote_user }} + IdentityFile {{ backup_script_ssh_key_location }} + StrictHostKeyChecking accept-new diff --git a/roles/backups/vars/main/defaults.yml b/roles/backups/vars/main/defaults.yml new file mode 100644 index 0000000..320628f --- /dev/null +++ b/roles/backups/vars/main/defaults.yml @@ -0,0 +1,4 @@ +--- +backup_script_ssh_key_location: /root/.ssh/id_ssh_rsa +backups_script_path: /usr/local/bin/backup_script.sh +backups_configuration_path: /etc/borg_backup_script/backup_configuration.yaml diff --git a/roles/backups/vars/main/vault.yml b/roles/backups/vars/main/vault.yml new file mode 100644 index 0000000..cffe045 --- /dev/null +++ b/roles/backups/vars/main/vault.yml @@ -0,0 +1,18 @@ +$ANSIBLE_VAULT;1.1;AES256 +36663737323462306164663362663633363838633165303464666233333364323330613933326237 +3565663233613037336633313537346534333432633036360a353064366464333164393161653038 +30633966613031363932633736333337653464373866333836353032356431393866303836343166 +6464333031323639660a306631363234383366643435366536323861356434393566643633643839 +35313064653536393366366536386331663062663132313331353238653933356234333338343436 +32616565323636633239346366323934303766353936653336353063373663623932353532386532 +32633736323866313133363438373639396663333737363536353731353236303333626364386632 +64363336356566653130303765396232646231333436366434353634316631313365373561383636 +38386636623265643762613065376362653964653935306338653763306137323165346332623264 +33636164613562636164363065623564363965626235643238363630666639363866663631643530 +65613938663131396630303565646335623764353830356536376465346339363034316666306134 +31353731316430663136613061386566613832626234656337343065363331636239326365343762 +33663965626538643937323832663638613766323331623133376632666131353936346238386437 +61306135386131653466633331313165626162306639323633383133643761633466373234353134 +39323237666334323232623230643734363765376163333762643962356365343364383939333132 +63363961383934643935323264326133313135336638323833336539393136306435663134333930 +32343762623636323637383530366434326537313431636131343533613733613063 diff --git a/roles/packages/tasks/main.yml b/roles/packages/tasks/main.yml index 5b58c6a..fcecb77 100644 --- a/roles/packages/tasks/main.yml +++ b/roles/packages/tasks/main.yml @@ -3,11 +3,15 @@ become: true ansible.builtin.apt: pkg: + - tldr + - tree - git - vim - dnsutils - rsyslog - snapd + - yq + - borgbackup state: present - name: Install Snap Core become: true