Use duplicity directly on volumes to improve performance and benefit from reduced incremental backup sizes.

backup_ovh1.yml

@@ -13,32 +13,15 @@
         path: "{{ WORKDIR }}/backup"
         state: directory
 
-    - name: Archive volumes
+    # python3-swiftclient is a requirement of duplicity
-      ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/{{ item }}.tar.gz -C /mnt/volumes {{ item }}"
-      with_items: "{{ BACKUP_OVH1_VOLUMES }}"
-
-    - name: Find lastest MySQL DB dump
-      ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump-mysql_*"
-      register: MySQL_dump
-
-    - name: Find lastest applications DB dump
-      ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump_*"
-      register: DBs_dump
-
-    - name: Archive DB dumps
-      ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/mysql-server_dumps.tar.gz -C /mnt/volumes {{ MySQL_dump.stdout_lines | last }} {{ DBs_dump.stdout_lines | last }}"
-      with_items: "{{ BACKUP_OVH1_VOLUMES }}"
-
-# python3-swiftclient is a requirement of duplicity
     - name: Install python3-swiftclient
       ansible.builtin.package:
         name: python3-swiftclient
         state: present
 
-    - name: Backup with duplicity
+    - name: Archive volume
-      ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch '{{ WORKDIR }}/backup' swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
+      ansible.builtin.include_tasks: "tasks/backup_volume.yml"
-      environment: "{{ DUPLICITY_ENVIRONMENT }}"
+      with_items: "{{ BACKUP_OVH1_VOLUMES }}"
 
-    - name: Clean old duplicity backups
+    - name: Archive MySQL DB
-      ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
+      ansible.builtin.include_tasks: "tasks/backup_mysql_db.yml"
-      environment: "{{ DUPLICITY_ENVIRONMENT }}"

tasks/backup_mysql_db.yml

@@ -0,0 +1,24 @@
+---
+- name: Create backup directory
+  ansible.builtin.file:
+    path: "{{ WORKDIR }}/backup/MySQL_DB"
+    state: directory
+
+- name: Find lastest MySQL DB dump
+  ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump-mysql_*"
+  register: MySQL_dump
+
+- name: Find lastest applications DB dump
+  ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump_*"
+  register: DBs_dump
+
+- name: Archive DB dumps
+  ansible.builtin.command: "cp -a /mnt/volumes/{{ MySQL_dump.stdout_lines | last }} /mnt/volumes/{{ DBs_dump.stdout_lines | last }} {{ WORKDIR }}/backup/MySQL_DB/"
+
+- name: Backup with duplicity
+  ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name MySQL_DB --allow-source-mismatch '{{ WORKDIR }}/backup/MySQL_DB' swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
+  environment: "{{ DUPLICITY_ENVIRONMENT }}"
+
+- name: Clean old duplicity backups
+  ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name MySQL_DB --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
+  environment: "{{ DUPLICITY_ENVIRONMENT }}"

tasks/backup_volume.yml

@@ -0,0 +1,19 @@
+---
+- name: Find hard links
+  ansible.builtin.command: "find '/mnt/volumes/{{ item }}' -type f -links +1"
+  register: find_hard_links
+
+# Duplicity does not support hard links
+- name: Assert that there is no hard links in /mnt/volumes/{{ item }}
+  ansible.builtin.assert:
+    that:
+      - find_hard_links.stdout | length = 0
+    msg: "Duplicity does not support hard links."
+
+- name: Backup with duplicity
+  ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ item }} --allow-source-mismatch '/mnt/volumes/{{ item }}' swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
+  environment: "{{ DUPLICITY_ENVIRONMENT }}"
+
+- name: Clean old duplicity backups
+  ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ item }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
+  environment: "{{ DUPLICITY_ENVIRONMENT }}"