yohan/duplicity_playbooks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: yohan:master
Branches Tags
yohan:master
yohan:dev
..
compare: yohan:dev
Branches Tags
yohan:master
yohan:dev

No commits in common. "master" and "dev" have entirely different histories.
master ... dev

10 changed files with 104 additions and 226 deletions
Show Stats Expand all files Collapse all files

35
backup_ovh1.yml
View File

@ -13,21 +13,32 @@
path: "{{ WORKDIR }}/backup"
state: directory
# python3-swiftclient is a requirement of duplicity
- name: Archive volumes
ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/{{ item }}.tar.gz -C /mnt/volumes {{ item }}"
with_items: "{{ BACKUP_OVH1_VOLUMES }}"
- name: Find lastest MySQL DB dump
ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump-mysql_*"
register: MySQL_dump
- name: Find lastest applications DB dump
ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump_*"
register: DBs_dump
- name: Archive DB dumps
ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/mysql-server_dumps.tar.gz -C /mnt/volumes {{ MySQL_dump.stdout_lines | last }} {{ DBs_dump.stdout_lines | last }}"
with_items: "{{ BACKUP_OVH1_VOLUMES }}"
# python3-swiftclient is a requirement of duplicity
- name: Install python3-swiftclient
ansible.builtin.package:
name: python3-swiftclient
state: present
- name: Backup standard volume
ansible.builtin.include_tasks: "tasks/backup_volume.yml"
with_items: "{{ BACKUP_OVH1_VOLUMES }}"
- name: Backup with duplicity
ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch '{{ WORKDIR }}/backup' swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
environment: "{{ DUPLICITY_ENVIRONMENT }}"
- name: Prepare MySQL_DB backup volume
ansible.builtin.include_tasks: "tasks/prepare_MySQL_DB_backup_volume.yml"
- name: Backup last MySQL dumps
ansible.builtin.include_tasks: "tasks/backup_volume.yml"
with_items:
- name: mysql-server_dumps
dir: "{{ WORKDIR }}/backup/MySQL_DB"
- name: Clean old duplicity backups
ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
environment: "{{ DUPLICITY_ENVIRONMENT }}"

39
bootstrap.yml
View File

@ -24,35 +24,14 @@
state: directory
mode: '0700'
- name: Extract required secrets from secrets.tar.gz.enc
shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in {{ WORKDIR }}/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 2 {{ item.name }}"
with_items:
- name: secrets/docker-duplicity-stack/mail_credentials.json
dir: /root/
- name: secrets/bootstrap/id_rsa
dir: /root/.ssh
- name: Change secret file permissions
ansible.builtin.file:
path: "{{ item }}"
mode: '0400'
owner: root
group: root
with_items:
- /root/mail_credentials.json
- /root/.ssh/id_rsa
- name: Copy ssh config
ansible.builtin.copy:
src: /root/duplicity_playbooks/files/config
dest: /root/.ssh/config
mode: '0640'
owner: root
group: root
- name: Extract from secrets.tar.gz.enc
shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in {{ WORKDIR }}/secrets.tar.gz.enc | tar -zxv -C {{ WORKDIR }}"
- name: Change SSH private key permissions
ansible.builtin.file:
path: /root/.ssh/id_rsa
mode: '0400'
- name: Retrieve documentation
ansible.builtin.get_url:
url: "https://{{ CLOUD_SERVER }}/s/{{ lookup('env','DOC_KEY') }}/download"
@ -78,14 +57,6 @@
ansible.builtin.package:
name: python2
state: present
when: copy_output is changed
- name: Copy sendmail.py
ansible.builtin.copy:
src: /root/duplicity_playbooks/files/sendmail.py
dest: /root/sendmail.py
mode: '0770'
when: copy_output is changed
- name: Send mail with new secrets
ansible.builtin.command: "/root/sendmail.py -a {{ WORKDIR }}/secrets.tar.gz.enc {{ WORKDIR }}/mail /root/mail_credentials.json"

1
files/config
View File

@ -1 +0,0 @@
StrictHostKeyChecking accept-new

91
files/sendmail.py
View File

@ -1,91 +0,0 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
import sys
import os
import json
from datetime import datetime
import logging
logging.basicConfig()
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("text_file", help="Please specify the file containing the message as plain text.")
parser.add_argument("credentials", help="Please specify the file containing the SMTP credentials as JSON text.")
parser.add_argument("-i", "--html_file", default=None, help="If needed, specify the file containing the message as HTML.")
parser.add_argument("-a", "--attachment", default=None, help="If needed, specify the attachment path.")
args = parser.parse_args()
with open(args.credentials) as json_file:
data = json.load(json_file)
user = data["user"]
password = data["password"]
sender = data["sender"]
receiver = data["receiver"]
# Create message container - the correct MIME type is
# multipart/alternative or multipart/mixed if there are attachments.
if args.attachment is not None:
msg = MIMEMultipart('mixed')
text_msg = MIMEMultipart('alternative')
else:
msg = MIMEMultipart('alternative')
msg['Subject'] = u"Secrets archive changed."
msg['From'] = sender
msg['To'] = receiver
with open(args.text_file, 'r') as file:
text = file.read()
#text = u"Bonjour , \n \
# Test."
part1 = MIMEText(text, 'plain', 'utf-8')
if args.attachment is not None:
text_msg.attach(part1)
else:
msg.attach(part1)
if args.html_file is not None:
with open(args.html_file, 'r') as file:
html = file.read()
# html = u"""\
# <html>
# <head></head>
# <body>
# <p>Bonjour,<br>
# </p>
# <p>Test
# </p>
# </body>
# </html>
# """
part2 = MIMEText(html, 'html', 'utf-8')
if args.attachment is not None:
text_msg.attach(part2)
else:
msg.attach(part2)
if args.attachment is not None:
msg.attach(text_msg)
with open(args.attachment, 'rb') as file:
attachment = MIMEApplication(file.read(), 'octet-stream')
attachment.add_header('Content-Disposition', 'attachment',
filename=os.path.basename(file.name))
msg.attach(attachment)
# print msg.as_string().encode('ascii')
print "sending"
s = smtplib.SMTP('ssl0.ovh.net', 587)
s.set_debuglevel(1)
s.login(user, password)
# sendmail function takes 3 arguments: sender's address, recipient's address
# and message to send - here it is sent as one string.
s.sendmail(sender, receiver, msg.as_string().encode('ascii'))
s.quit()

2
launch_ansible_container.sh
View File

@ -6,4 +6,4 @@ SCRIPTPATH=$(dirname $SCRIPT)
cd $SCRIPTPATH
USER=$(whoami)
sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -e ANSIBLE_VERBOSITY -v $SCRIPTPATH:/root/duplicity_playbooks -i ansible /root/duplicity_playbooks/launch_top_playbook.sh
sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -v $SCRIPTPATH:/root/duplicity_playbooks -i ansible /root/duplicity_playbooks/launch_top_playbook.sh

26
tasks/backup_volume.yml
View File

@ -1,26 +0,0 @@
---
- name: Find hard links in {{ item.dir }}
ansible.builtin.command: "find '{{ item.dir }}' -type f -links +1"
register: find_hard_links
# Duplicity does not support hard links
- name: Assert that there is no hard links in {{ item.dir }}
ansible.builtin.assert:
that:
- find_hard_links.stdout | length == 0
msg: "Duplicity does not support hard links."
- name: Create SWIFT bucket {{ item.name }}
openstack.cloud.object_container:
name: "{{ item.name }}"
state: present
environment: "{{ DUPLICITY_ENVIRONMENT }}"
- name: Backup {{ item.dir }} with duplicity
ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ item.name }} --allow-source-mismatch '{{ item.dir }}' swift://{{ item.name }}"
environment: "{{ DUPLICITY_ENVIRONMENT }}"
- name: Clean old duplicity backups for {{ item.name }}
ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ item.name }} --allow-source-mismatch --force swift://{{ item.name }}"
environment: "{{ DUPLICITY_ENVIRONMENT }}"

8
tasks/down.yml
View File

@ -13,6 +13,14 @@
delegate_to: 172.17.0.1
become: true
- name: Remove docker-duplicity-stack directory
ansible.builtin.file:
path: "/home/{{ user }}/repository/docker-duplicity-stack"
state: absent
remote_user: "{{ user }}"
delegate_to: 172.17.0.1
become: true
- name: unmount /mnt/cloud
ansible.posix.mount:
path: /mnt/cloud

16
tasks/prepare_MySQL_DB_backup_volume.yml
View File

@ -1,16 +0,0 @@
---
- name: Create backup directory
ansible.builtin.file:
path: "{{ WORKDIR }}/backup/MySQL_DB"
state: directory
- name: Find lastest MySQL DB dump
ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump-mysql_*"
register: MySQL_dump
- name: Find lastest applications DB dump
ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump_*"
register: DBs_dump
- name: Copy last DB dumps in backup directory
ansible.builtin.command: "cp -a /mnt/volumes/{{ MySQL_dump.stdout_lines | last }} /mnt/volumes/{{ DBs_dump.stdout_lines | last }} {{ WORKDIR }}/backup/MySQL_DB/"

68
tasks/start.yml
View File

@ -22,15 +22,13 @@
- name: Extract from secrets.tar.gz.enc
shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in /root/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 2 {{ item.name }}"
with_items:
- name: secrets/docker-duplicity-stack/mail_credentials.json
dir: /root/
- name: secrets/bootstrap/id_rsa
dir: /root/.ssh
- name: secrets/docker-duplicity-stack/nextcloud_password.sh
dir: /root
- name: Extract secrets.yml from secrets.tar.gz.enc
shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in /root/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 1 {{ item.name }}"
with_items:
- name: secrets/secrets.yml
- name: secrets/bootstrap/openrc.sh
dir: /root
- name: Change SSH private key permissions
@ -38,6 +36,36 @@
path: /root/.ssh/id_rsa
mode: '0400'
- name: Remove docker-duplicity-stack directory
ansible.builtin.file:
path: "/home/{{ user }}/repository/docker-duplicity-stack"
state: absent
remote_user: "{{ user }}"
delegate_to: 172.17.0.1
become: true
- name: Clone docker-duplicity-stack repo
ansible.builtin.git:
repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity-stack.git'
dest: "/home/{{ user }}/repository/docker-duplicity-stack"
clone: yes
force: true
remote_user: "{{ user }}"
delegate_to: 172.17.0.1
become: true
- name: Copy files
ansible.builtin.copy:
src: "{{ item }}"
dest: "/home/{{ user }}/repository/docker-duplicity-stack"
mode: '0400'
remote_user: "{{ user }}"
delegate_to: 172.17.0.1
become: true
with_items:
- /root/mail_credentials.json
- /root/.ssh/id_rsa
- name: Set Nextcloud credentials
ansible.builtin.include_tasks: "tasks/source_vars.yml"
with_items:
@ -84,8 +112,20 @@
delegate_to: 172.17.0.1
become: true
- name: Include secrets from yml db
ansible.builtin.include_vars: "/root/secrets.yml"
- name: Set OpenStack credentials
ansible.builtin.include_tasks: "tasks/source_vars.yml"
with_items:
- OS_AUTH_URL
- OS_IDENTITY_API_VERSION
- OS_USER_DOMAIN_NAME
- OS_PROJECT_DOMAIN_NAME
- OS_TENANT_ID
- OS_TENANT_NAME
- OS_USERNAME
- OS_PASSWORD
- OS_REGION_NAME
vars:
shell_script: /root/openrc.sh
- name: Setup volume
ansible.builtin.include_tasks: "tasks/setup_volume.yml"
@ -105,7 +145,7 @@
repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git'
clone: no
update: no
version: master
version: dev
register: git
- name: Set fact tag
@ -136,7 +176,7 @@
repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git'
dest: "/home/{{ user }}/build_docker-duplicity"
clone: yes
version: master
version: dev
when:
- local_duplicity_image.msg is defined
- '"Cannot find the image" in local_duplicity_image.msg'
@ -179,12 +219,11 @@
repo: 'https://{{ GIT_SERVER }}/yohan/duplicity_playbooks.git'
dest: "/home/{{ user }}/repository/duplicity_playbooks_temp"
clone: yes
version: master
version: dev
force: true
remote_user: "{{ user }}"
delegate_to: 172.17.0.1
# We are forced to use another container because mounts done on the hosts after current container start will not be visible
- name: Start duplicity container
community.docker.docker_container:
name: duplicity
@ -195,9 +234,15 @@
output_logs: true
detach: false
network_mode: host
working_dir: "/home/{{ user }}/repository/docker-duplicity-stack"
volumes:
- /mnt/volumes:/mnt/volumes:z
- /mnt/cloud:/mnt/cloud:z
- /home/{{ user }}/repository/docker-duplicity-stack/backup_scripts:/mnt/scripts:z
- /home/{{ user }}/repository/docker-duplicity-stack/sendmail.py:/root/sendmail.py:z
- /home/{{ user }}/repository/docker-duplicity-stack/mail_credentials.json:/root/mail_credentials.json:z
- /home/{{ user }}/repository/docker-duplicity-stack/id_rsa:/root/.ssh/id_rsa:Z
- /home/{{ user }}/repository/docker-duplicity-stack/config:/root/.ssh/config:Z
- /home/{{ user }}/repository/duplicity_playbooks_temp:/root/duplicity_playbooks:Z
env:
OS_AUTH_URL: "{{ OS_AUTH_URL }}"
@ -214,7 +259,6 @@
SECRETS_ARCHIVE_PASSPHRASE: "{{ lookup('env','SECRETS_ARCHIVE_PASSPHRASE') }}"
DUPLICITY_PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}"
BACKUP_WORKFLOW: "{{ lookup('env','BACKUP_WORKFLOW') }}"
ANSIBLE_VERBOSITY: "{{ lookup('env','ANSIBLE_VERBOSITY') }}"
remote_user: "{{ user }}"
delegate_to: 172.17.0.1
become: true

44
vars/main.yml
View File

@ -12,6 +12,7 @@ BOOTSTRAP_REPOS:
- docker-gogs-stack
- docker-mysql-stack
- docker-mysql
- systemd-mount-cinder-volume
BOOTSTRAP_REQUIRED_ENV_VARS:
- OS_AUTH_URL
@ -27,38 +28,15 @@ BOOTSTRAP_REQUIRED_ENV_VARS:
- BACKUP_WORKFLOW
BACKUP_OVH1_VOLUMES:
- name: elasticsearch_data
dir: /mnt/volumes/elasticsearch_data/data
- name: gogs_data
dir: /mnt/volumes/gogs_data/data
- name: mail_data
dir: /mnt/volumes/mail_data/data
- name: mail_data_Sylvie
dir: /mnt/volumes/mail_data_Sylvie/data
- name: nextcloud
dir: /mnt/volumes/nextcloud/data
- name: reverse-proxy_conf
dir: /mnt/volumes/reverse-proxy_conf/data
- name: reverse-proxy_conf_enabled
dir: /mnt/volumes/reverse-proxy_conf_enabled/data
- name: reverse-proxy_letsencrypt
dir: /mnt/volumes/reverse-proxy_letsencrypt/data
- name: scuttle_code
dir: /mnt/volumes/scuttle_code/data
- name: scuttle_php5-fpm_conf
dir: /mnt/volumes/scuttle_php5-fpm_conf/data
- name: etc_grafana
dir: /mnt/volumes/etc_grafana/data
- name: var_lib_grafana
dir: /mnt/volumes/var_lib_grafana/data
- name: var_log_grafana
dir: /mnt/volumes/var_log_grafana/data
- name: registry_data
dir: /mnt/volumes/registry_data/data
- name: registry_auth
dir: /mnt/volumes/registry_auth/data
- name: registry_certs
dir: /mnt/volumes/registry_certs/data
- elasticsearch_data
- gogs_data
- mail_data
- nextcloud
- reverse-proxy_conf
- reverse-proxy_conf_enabled
- reverse-proxy_letsencrypt
- scuttle_code
- scuttle_php5-fpm_conf
BACKUP_OVH1_REQUIRED_ENV_VARS:
- OS_AUTH_URL
@ -83,6 +61,6 @@ DUPLICITY_ENVIRONMENT:
SWIFT_TENANTNAME: "{{ lookup('env','OS_TENANT_NAME') }}"
SWIFT_USERNAME: "{{ lookup('env','OS_USERNAME') }}"
SWIFT_PASSWORD: "{{ lookup('env','OS_PASSWORD') }}"
SWIFT_REGIONNAME: "{{ lookup('env','OS_REGION_NAME') }}"
SWIFT_REGION_NAME: "{{ lookup('env','OS_REGION_NAME') }}"
PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}"