Fix secrets.

This reverts commit c5d46259d8.

backup_ovh1.yml

@@ -13,32 +13,21 @@
         path: "{{ WORKDIR }}/backup"
         state: directory
 
-    - name: Archive volumes
-      ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/{{ item }}.tar.gz -C /mnt/volumes {{ item }}"
-      with_items: "{{ BACKUP_OVH1_VOLUMES }}"
-
-    - name: Find lastest MySQL DB dump
-      ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump-mysql_*"
-      register: MySQL_dump
-
-    - name: Find lastest applications DB dump
-      ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump_*"
-      register: DBs_dump
-
-    - name: Archive DB dumps
-      ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/mysql-server_dumps.tar.gz -C /mnt/volumes {{ MySQL_dump.stdout_lines | last }} {{ DBs_dump.stdout_lines | last }}"
-      with_items: "{{ BACKUP_OVH1_VOLUMES }}"
-
-# python3-swiftclient is a requirement of duplicity
+    # python3-swiftclient is a requirement of duplicity
     - name: Install python3-swiftclient
       ansible.builtin.package:
         name: python3-swiftclient
         state: present
 
-    - name: Backup with duplicity
-      ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch '{{ WORKDIR }}/backup' swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
-      environment: "{{ DUPLICITY_ENVIRONMENT }}"
+    - name: Backup standard volume
+      ansible.builtin.include_tasks: "tasks/backup_volume.yml"
+      with_items: "{{ BACKUP_OVH1_VOLUMES }}"
 
-    - name: Clean old duplicity backups
-      ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}"
-      environment: "{{ DUPLICITY_ENVIRONMENT }}"
+    - name: Prepare MySQL_DB backup volume
+      ansible.builtin.include_tasks: "tasks/prepare_MySQL_DB_backup_volume.yml"
+
+    - name: Backup last MySQL dumps
+      ansible.builtin.include_tasks: "tasks/backup_volume.yml"
+      with_items:
+        - name: mysql-server_dumps
+          dir: "{{ WORKDIR }}/backup/MySQL_DB"

bootstrap.yml

@@ -24,14 +24,35 @@
         state: directory
         mode: '0700'
     
+    - name: Extract required secrets from secrets.tar.gz.enc
+      shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in {{ WORKDIR }}/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 2 {{ item.name }}"
+      with_items:
+        - name: secrets/docker-duplicity-stack/mail_credentials.json
+          dir: /root/
+        - name: secrets/bootstrap/id_rsa
+          dir: /root/.ssh
+
+    - name: Change secret file permissions
+      ansible.builtin.file:
+        path: "{{ item }}"
+        mode: '0400'
+        owner: root
+        group: root
+      with_items:
+        - /root/mail_credentials.json
+        - /root/.ssh/id_rsa
+
+    - name: Copy ssh config
+      ansible.builtin.copy:
+        src: /root/duplicity_playbooks/files/config
+        dest: /root/.ssh/config
+        mode: '0640'
+        owner: root
+        group: root
+
     - name: Extract from secrets.tar.gz.enc
       shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in {{ WORKDIR }}/secrets.tar.gz.enc | tar -zxv -C {{ WORKDIR }}"
 
-    - name: Change SSH private key permissions
-      ansible.builtin.file:
-        path: /root/.ssh/id_rsa
-        mode: '0400'
-
     - name: Retrieve documentation
       ansible.builtin.get_url:
         url: "https://{{ CLOUD_SERVER }}/s/{{ lookup('env','DOC_KEY') }}/download"
@@ -57,6 +78,14 @@
       ansible.builtin.package:
         name: python2
         state: present
+      when: copy_output is changed
+
+    - name: Copy sendmail.py
+      ansible.builtin.copy:
+        src: /root/duplicity_playbooks/files/sendmail.py
+        dest: /root/sendmail.py
+        mode: '0770'
+      when: copy_output is changed
 
     - name: Send mail with new secrets
       ansible.builtin.command: "/root/sendmail.py -a {{ WORKDIR }}/secrets.tar.gz.enc {{ WORKDIR }}/mail /root/mail_credentials.json"

files/config

@@ -0,0 +1 @@
+StrictHostKeyChecking accept-new

files/sendmail.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import smtplib
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.application import MIMEApplication
+import sys
+import os
+import json
+from datetime import datetime
+import logging
+logging.basicConfig()
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("text_file", help="Please specify the file containing the message as plain text.")
+parser.add_argument("credentials", help="Please specify the file containing the SMTP credentials as JSON text.")
+parser.add_argument("-i", "--html_file", default=None, help="If needed, specify the file containing the message as HTML.")
+parser.add_argument("-a", "--attachment", default=None, help="If needed, specify the attachment path.")
+
+args = parser.parse_args()
+
+with open(args.credentials) as json_file:
+    data = json.load(json_file)
+    user = data["user"]
+    password = data["password"]
+    sender = data["sender"]
+    receiver = data["receiver"]
+
+# Create message container - the correct MIME type is
+# multipart/alternative or multipart/mixed if there are attachments.
+if args.attachment is not None:
+    msg = MIMEMultipart('mixed')
+    text_msg = MIMEMultipart('alternative')
+else:
+    msg = MIMEMultipart('alternative')
+msg['Subject'] = u"Secrets archive changed."
+msg['From'] = sender
+msg['To'] = receiver
+
+with open(args.text_file, 'r') as file:
+    text = file.read()
+#text = u"Bonjour , \n \
+#	Test."
+part1 = MIMEText(text, 'plain', 'utf-8')
+if args.attachment is not None:
+    text_msg.attach(part1)
+else:
+    msg.attach(part1)
+if args.html_file is not None:
+    with open(args.html_file, 'r') as file:
+        html = file.read()
+
+#    html = u"""\
+#	<html>
+#	  <head></head>
+#	  <body>
+#	    <p>Bonjour,<br>
+#	    </p>
+#	    <p>Test
+#	    </p>
+#	  </body>
+#	</html>
+#	"""
+    part2 = MIMEText(html, 'html', 'utf-8')
+    if args.attachment is not None:
+        text_msg.attach(part2)
+    else:
+        msg.attach(part2)
+
+if args.attachment is not None:
+    msg.attach(text_msg)
+    with open(args.attachment, 'rb') as file:
+        attachment = MIMEApplication(file.read(), 'octet-stream')
+    attachment.add_header('Content-Disposition', 'attachment',
+                        filename=os.path.basename(file.name))
+    msg.attach(attachment)
+
+# print msg.as_string().encode('ascii')
+print "sending"
+s = smtplib.SMTP('ssl0.ovh.net', 587)
+s.set_debuglevel(1)
+s.login(user, password)
+
+# sendmail function takes 3 arguments: sender's address, recipient's address
+# and message to send - here it is sent as one string.
+s.sendmail(sender, receiver, msg.as_string().encode('ascii'))
+s.quit()
+

launch_ansible_container.sh

@@ -6,4 +6,4 @@ SCRIPTPATH=$(dirname $SCRIPT)
 
 cd $SCRIPTPATH
 USER=$(whoami)
-sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -v $SCRIPTPATH:/root/duplicity_playbooks -i ansible /root/duplicity_playbooks/launch_top_playbook.sh
+sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -e ANSIBLE_VERBOSITY -v $SCRIPTPATH:/root/duplicity_playbooks -i ansible /root/duplicity_playbooks/launch_top_playbook.sh

tasks/backup_volume.yml

@@ -0,0 +1,26 @@
+---
+- name: Find hard links in {{ item.dir }}
+  ansible.builtin.command: "find '{{ item.dir }}' -type f -links +1"
+  register: find_hard_links
+
+# Duplicity does not support hard links
+- name: Assert that there is no hard links in {{ item.dir }}
+  ansible.builtin.assert:
+    that:
+      - find_hard_links.stdout | length == 0
+    msg: "Duplicity does not support hard links."
+
+- name: Create SWIFT bucket {{ item.name }}
+  openstack.cloud.object_container:
+    name: "{{ item.name }}"
+    state: present
+  environment: "{{ DUPLICITY_ENVIRONMENT }}"
+
+- name: Backup {{ item.dir }} with duplicity
+  ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ item.name }} --allow-source-mismatch '{{ item.dir }}' swift://{{ item.name }}"
+  environment: "{{ DUPLICITY_ENVIRONMENT }}"
+
+- name: Clean old duplicity backups for {{ item.name }}
+  ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ item.name }} --allow-source-mismatch --force swift://{{ item.name }}"
+  environment: "{{ DUPLICITY_ENVIRONMENT }}"
+

tasks/down.yml

@@ -13,14 +13,6 @@
   delegate_to: 172.17.0.1
   become: true
 
-- name: Remove docker-duplicity-stack directory
-  ansible.builtin.file:
-    path: "/home/{{ user }}/repository/docker-duplicity-stack"
-    state: absent
-  remote_user: "{{ user }}"
-  delegate_to: 172.17.0.1
-  become: true
-
 - name: unmount /mnt/cloud
   ansible.posix.mount:
     path: /mnt/cloud

tasks/prepare_MySQL_DB_backup_volume.yml

@@ -0,0 +1,16 @@
+---
+- name: Create backup directory
+  ansible.builtin.file:
+    path: "{{ WORKDIR }}/backup/MySQL_DB"
+    state: directory
+
+- name: Find lastest MySQL DB dump
+  ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump-mysql_*"
+  register: MySQL_dump
+
+- name: Find lastest applications DB dump
+  ansible.builtin.shell: "cd /mnt/volumes; ls -tr mysql-server_dumps/data/mysql_dump_*"
+  register: DBs_dump
+
+- name: Copy last DB dumps in backup directory
+  ansible.builtin.command: "cp -a /mnt/volumes/{{ MySQL_dump.stdout_lines | last }} /mnt/volumes/{{ DBs_dump.stdout_lines | last }} {{ WORKDIR }}/backup/MySQL_DB/"

tasks/start.yml

@@ -22,13 +22,15 @@
 - name: Extract from secrets.tar.gz.enc
   shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in /root/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 2 {{ item.name }}"
   with_items:
-    - name: secrets/docker-duplicity-stack/mail_credentials.json
-      dir: /root/
     - name: secrets/bootstrap/id_rsa
       dir: /root/.ssh
     - name: secrets/docker-duplicity-stack/nextcloud_password.sh
       dir: /root
-    - name: secrets/bootstrap/openrc.sh
+
+- name: Extract secrets.yml from secrets.tar.gz.enc
+  shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in /root/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 1 {{ item.name }}"
+  with_items:
+    - name: secrets/secrets.yml
       dir: /root
 
 - name: Change SSH private key permissions
@@ -36,36 +38,6 @@
     path: /root/.ssh/id_rsa
     mode: '0400'
 
-- name: Remove docker-duplicity-stack directory
-  ansible.builtin.file:
-    path: "/home/{{ user }}/repository/docker-duplicity-stack"
-    state: absent
-  remote_user: "{{ user }}"
-  delegate_to: 172.17.0.1
-  become: true
-
-- name: Clone docker-duplicity-stack repo
-  ansible.builtin.git:
-    repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity-stack.git'
-    dest: "/home/{{ user }}/repository/docker-duplicity-stack"
-    clone: yes
-    force: true
-  remote_user: "{{ user }}"
-  delegate_to: 172.17.0.1
-  become: true
-
-- name: Copy files
-  ansible.builtin.copy:
-    src: "{{ item }}"
-    dest: "/home/{{ user }}/repository/docker-duplicity-stack"
-    mode: '0400'
-  remote_user: "{{ user }}"
-  delegate_to: 172.17.0.1
-  become: true
-  with_items:
-    - /root/mail_credentials.json
-    - /root/.ssh/id_rsa
-
 - name: Set Nextcloud credentials
   ansible.builtin.include_tasks: "tasks/source_vars.yml"
   with_items:
@@ -112,20 +84,8 @@
   delegate_to: 172.17.0.1
   become: true
 
-- name: Set OpenStack credentials
-  ansible.builtin.include_tasks: "tasks/source_vars.yml"
-  with_items:
-    - OS_AUTH_URL
-    - OS_IDENTITY_API_VERSION
-    - OS_USER_DOMAIN_NAME
-    - OS_PROJECT_DOMAIN_NAME
-    - OS_TENANT_ID
-    - OS_TENANT_NAME
-    - OS_USERNAME
-    - OS_PASSWORD
-    - OS_REGION_NAME
-  vars:
-    shell_script: /root/openrc.sh
+- name: Include secrets from yml db
+  ansible.builtin.include_vars: "/root/secrets.yml"
 
 - name: Setup volume
   ansible.builtin.include_tasks: "tasks/setup_volume.yml"
@@ -145,7 +105,7 @@
     repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git'
     clone: no
     update: no
-    version: dev
+    version: master
   register: git
 
 - name: Set fact tag
@@ -176,7 +136,7 @@
     repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git'
     dest: "/home/{{ user }}/build_docker-duplicity"
     clone: yes
-    version: dev
+    version: master
   when:
     - local_duplicity_image.msg is defined
     - '"Cannot find the image" in local_duplicity_image.msg'
@@ -219,11 +179,12 @@
     repo: 'https://{{ GIT_SERVER }}/yohan/duplicity_playbooks.git'
     dest: "/home/{{ user }}/repository/duplicity_playbooks_temp"
     clone: yes
-    version: dev
+    version: master
     force: true
   remote_user: "{{ user }}"
   delegate_to: 172.17.0.1
 
+# We are forced to use another container because mounts done on the hosts after current container start will not be visible
 - name: Start duplicity container
   community.docker.docker_container:
     name: duplicity
@@ -234,15 +195,9 @@
     output_logs: true
     detach: false
     network_mode: host
-    working_dir: "/home/{{ user }}/repository/docker-duplicity-stack"
     volumes:
       - /mnt/volumes:/mnt/volumes:z
       - /mnt/cloud:/mnt/cloud:z
-      - /home/{{ user }}/repository/docker-duplicity-stack/backup_scripts:/mnt/scripts:z
-      - /home/{{ user }}/repository/docker-duplicity-stack/sendmail.py:/root/sendmail.py:z
-      - /home/{{ user }}/repository/docker-duplicity-stack/mail_credentials.json:/root/mail_credentials.json:z
-      - /home/{{ user }}/repository/docker-duplicity-stack/id_rsa:/root/.ssh/id_rsa:Z
-      - /home/{{ user }}/repository/docker-duplicity-stack/config:/root/.ssh/config:Z
       - /home/{{ user }}/repository/duplicity_playbooks_temp:/root/duplicity_playbooks:Z
     env:
       OS_AUTH_URL: "{{ OS_AUTH_URL }}"
@@ -259,6 +214,7 @@
       SECRETS_ARCHIVE_PASSPHRASE: "{{ lookup('env','SECRETS_ARCHIVE_PASSPHRASE') }}"
       DUPLICITY_PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}"
       BACKUP_WORKFLOW: "{{ lookup('env','BACKUP_WORKFLOW') }}"
+      ANSIBLE_VERBOSITY: "{{ lookup('env','ANSIBLE_VERBOSITY') }}"
   remote_user: "{{ user }}"
   delegate_to: 172.17.0.1
   become: true

vars/main.yml

@@ -12,7 +12,6 @@ BOOTSTRAP_REPOS:
   - docker-gogs-stack
   - docker-mysql-stack
   - docker-mysql
-  - systemd-mount-cinder-volume
 
 BOOTSTRAP_REQUIRED_ENV_VARS:
   - OS_AUTH_URL
@@ -28,15 +27,38 @@ BOOTSTRAP_REQUIRED_ENV_VARS:
   - BACKUP_WORKFLOW
 
 BACKUP_OVH1_VOLUMES:
-  - elasticsearch_data
-  - gogs_data
-  - mail_data
-  - nextcloud
-  - reverse-proxy_conf
-  - reverse-proxy_conf_enabled
-  - reverse-proxy_letsencrypt
-  - scuttle_code
-  - scuttle_php5-fpm_conf
+  - name: elasticsearch_data
+    dir: /mnt/volumes/elasticsearch_data/data
+  - name: gogs_data
+    dir: /mnt/volumes/gogs_data/data
+  - name: mail_data
+    dir: /mnt/volumes/mail_data/data
+  - name: mail_data_Sylvie
+    dir: /mnt/volumes/mail_data_Sylvie/data
+  - name: nextcloud
+    dir: /mnt/volumes/nextcloud/data
+  - name: reverse-proxy_conf
+    dir: /mnt/volumes/reverse-proxy_conf/data
+  - name: reverse-proxy_conf_enabled
+    dir: /mnt/volumes/reverse-proxy_conf_enabled/data
+  - name: reverse-proxy_letsencrypt
+    dir: /mnt/volumes/reverse-proxy_letsencrypt/data
+  - name: scuttle_code
+    dir: /mnt/volumes/scuttle_code/data
+  - name: scuttle_php5-fpm_conf
+    dir: /mnt/volumes/scuttle_php5-fpm_conf/data
+  - name: etc_grafana
+    dir: /mnt/volumes/etc_grafana/data
+  - name: var_lib_grafana
+    dir: /mnt/volumes/var_lib_grafana/data
+  - name: var_log_grafana
+    dir: /mnt/volumes/var_log_grafana/data
+  - name: registry_data
+    dir: /mnt/volumes/registry_data/data
+  - name: registry_auth
+    dir: /mnt/volumes/registry_auth/data
+  - name: registry_certs
+    dir: /mnt/volumes/registry_certs/data
 
 BACKUP_OVH1_REQUIRED_ENV_VARS:
   - OS_AUTH_URL
@@ -61,6 +83,6 @@ DUPLICITY_ENVIRONMENT:
   SWIFT_TENANTNAME: "{{ lookup('env','OS_TENANT_NAME') }}"
   SWIFT_USERNAME: "{{ lookup('env','OS_USERNAME') }}"
   SWIFT_PASSWORD: "{{ lookup('env','OS_PASSWORD') }}"
-  SWIFT_REGION_NAME: "{{ lookup('env','OS_REGION_NAME') }}"
+  SWIFT_REGIONNAME: "{{ lookup('env','OS_REGION_NAME') }}"
   PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}"