Bonjour,
-#
Test -#
-# -# -# """ - part2 = MIMEText(html, 'html', 'utf-8') - if args.attachment is not None: - text_msg.attach(part2) - else: - msg.attach(part2) - -if args.attachment is not None: - msg.attach(text_msg) - with open(args.attachment, 'rb') as file: - attachment = MIMEApplication(file.read(), 'octet-stream') - attachment.add_header('Content-Disposition', 'attachment', - filename=os.path.basename(file.name)) - msg.attach(attachment) - -# print msg.as_string().encode('ascii') -print "sending" -s = smtplib.SMTP('ssl0.ovh.net', 587) -s.set_debuglevel(1) -s.login(user, password) - -# sendmail function takes 3 arguments: sender's address, recipient's address -# and message to send - here it is sent as one string. -s.sendmail(sender, receiver, msg.as_string().encode('ascii')) -s.quit() diff --git a/launch_ansible_container.sh b/launch_ansible_container.sh index 04ec5bc..aae0a1f 100755 --- a/launch_ansible_container.sh +++ b/launch_ansible_container.sh @@ -6,4 +6,4 @@ SCRIPTPATH=$(dirname $SCRIPT) cd $SCRIPTPATH USER=$(whoami) -sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -v $SCRIPTPATH:/root/duplicity_playbooks -v /mnt:/mnt:z -i duplicity:master /root/duplicity_playbooks/launch_top_playbook.sh +sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -v $SCRIPTPATH:/root/duplicity_playbooks -i ansible /root/duplicity_playbooks/launch_top_playbook.sh diff --git a/playbook.yml b/playbook.yml index f2a34f5..84ce5f3 100755 --- a/playbook.yml +++ b/playbook.yml @@ -7,17 +7,5 @@ - name: Include start.yml ansible.builtin.include_tasks: "tasks/start.yml" - - name: Include {{ lookup('env','BACKUP_WORKFLOW') }}.yml - ansible.builtin.include_tasks: "tasks/{{ lookup('env','BACKUP_WORKFLOW') }}.yml" - vars: - DUPLICITY_ENVIRONMENT: - SWIFT_AUTHURL: "{{ OS_AUTH_URL }}" - SWIFT_AUTHVERSION: "{{ OS_IDENTITY_API_VERSION }}" - SWIFT_TENANTNAME: "{{ OS_TENANT_NAME }}" - SWIFT_USERNAME: "{{ OS_USERNAME }}" - SWIFT_PASSWORD: "{{ OS_PASSWORD }}" - SWIFT_REGION_NAME: GRA - PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}" - - name: Include down.yml ansible.builtin.include_tasks: "tasks/down.yml" diff --git a/tasks/assert_var_not_empty.yml b/tasks/assert_var_not_empty.yml deleted file mode 100755 index 0e7dde3..0000000 --- a/tasks/assert_var_not_empty.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Assert {{ item }} variable is set - ansible.builtin.assert: - that: - - item | length > 0 - msg: "{{ item }} variable must be set" diff --git a/tasks/backup_ovh1.yml b/tasks/backup_ovh1.yml deleted file mode 100755 index 8df339f..0000000 --- a/tasks/backup_ovh1.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Assert environment variable is not empty - ansible.builtin.include_tasks: "tasks/assert_env_var_not_empty.yml" - with_items: "{{ BACKUP_OVH1_REQUIRED_ENV_VARS }}" - -- name: Assert variable is not empty - ansible.builtin.include_tasks: "tasks/assert_var_not_empty.yml" - with_items: "{{ BACKUP_OVH1_REQUIRED_VARS }}" - -- name: Create backup directory - ansible.builtin.file: - path: "{{ WORKDIR }}/backup" - state: directory - -- name: Archive volumes - ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/{{ item }}.tar.gz -C /mnt/volumes {{ item }}" - with_items: "{{ BACKUP_OVH1_VOLUMES }}" - -- name: Find lastest MySQL DB dump - ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump-mysql_*" - register: MySQL_dump - -- name: Find lastest applications DB dump - ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump_*" - register: DBs_dump - -- name: Archive DB dumps - ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/mysql-server_dumps.tar.gz -C /mnt/volumes {{ MySQL_dump.stdout_lines | last }} {{ DBs_dump.stdout_lines | last }}" - with_items: "{{ BACKUP_OVH1_VOLUMES }}" - -thon3-swiftclient is a requirement of duplicity -- name: Install python3-swiftclient - ansible.builtin.package: - name: python3-swiftclient - state: present - -- name: Backup with duplicity - ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch '{{ WORKDIR }}/backup' swift://{{ lookup('env','BACKUP_WORKFLOW') }}" - environment: "{{ DUPLICITY_ENVIRONMENT }}" - -- name: Clean old duplicity backups - ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}" - environment: "{{ DUPLICITY_ENVIRONMENT }}" diff --git a/tasks/bootstrap.yml b/tasks/bootstrap.yml deleted file mode 100755 index 30fb74a..0000000 --- a/tasks/bootstrap.yml +++ /dev/null @@ -1,128 +0,0 @@ ---- -- name: Assert environment variable is not empty - ansible.builtin.include_tasks: "tasks/assert_env_var_not_empty.yml" - with_items: "{{ BOOTSTRAP_REQUIRED_ENV_VARS }}" - -- name: Assert variable is not empty - ansible.builtin.include_tasks: "tasks/assert_var_not_empty.yml" - with_items: "{{ BOOTSTRAP_REQUIRED_VARS }}" - -- name: Download secrets.tar.gz.enc - ansible.builtin.get_url: - url: "https://{{ CLOUD_SERVER }}/s/{{ lookup('env','KEY') }}/download?path=%2F&files=secrets.tar.gz.enc" - dest: "{{ WORKDIR }}/secrets.tar.gz.enc" - -- name: Install openssh-client - ansible.builtin.package: - name: openssh-client - state: present - -- name: Create /root/.ssh directory - ansible.builtin.file: - path: /root/.ssh - state: directory - mode: '0700' - -- name: Extract from secrets.tar.gz.enc - shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in {{ WORKDIR }}/secrets.tar.gz.enc | tar -zxv -C {{ WORKDIR }}" - -- name: Change SSH private key permissions - ansible.builtin.file: - path: /root/.ssh/id_rsa - mode: '0400' - -- name: Retrieve documentation - ansible.builtin.get_url: - url: "https://{{ CLOUD_SERVER }}/s/{{ lookup('env','DOC_KEY') }}/download" - dest: "{{ WORKDIR }}/Documentation.md" - -- name: Copy new documentation - ansible.builtin.copy: - src: "{{ WORKDIR }}/Documentation.md" - dest: "{{ WORKDIR }}/secrets/bootstrap/Documentation.md" - register: copy_output - -- name: Create secrets.tar.gz.enc - shell: "tar -czvpf - -C {{ WORKDIR }} secrets | openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -salt -out {{ WORKDIR }}/secrets.tar.gz.enc" - when: copy_output is changed - -- name: Copy mail content - ansible.builtin.copy: - content: "Secrets archive has changed. New file attached." - dest: "{{ WORKDIR }}/mail" - when: copy_output is changed - -- name: Install python2 - ansible.builtin.package: - name: python2 - state: present - -- name: Send mail with new secrets - ansible.builtin.command: "files/sendmail.py -a {{ WORKDIR }}/secrets.tar.gz.enc {{ WORKDIR }}/mail /root/mail_credentials.json" - when: copy_output is changed - -- name: Copy new secrets in Nextcloud share - ansible.builtin.copy: - src: "{{ WORKDIR }}/secrets.tar.gz.enc" - dest: /mnt/cloud/Passwords/secrets.tar.gz.enc - when: copy_output is changed - -- name: Create /mnt/archives_critiques/secrets directory on serveur-appart - ansible.builtin.file: - path: /mnt/archives_critiques/secrets - state: directory - owner: yohan - group: yohan - remote_user: yohan - vars: - ansible_ssh_port: 2224 - delegate_to: chez-yohan.scimetis.net - become: true - -- name: Get checksum of secrets.tar.gz.enc - ansible.builtin.stat: - path: "{{ WORKDIR }}/secrets.tar.gz.enc" - register: stats_output - -- name: Copy new secrets on serveur-appart - ansible.builtin.copy: - src: "{{ WORKDIR }}/secrets.tar.gz.enc" - dest: "/mnt/archives_critiques/secrets/secrets.tar.gz.enc-{{ stats_output.stat.checksum }}" - remote_user: yohan - vars: - ansible_ssh_port: 2224 - delegate_to: chez-yohan.scimetis.net - -- name: Clone repo - ansible.builtin.git: - repo: 'https://{{ GIT_SERVER }}/yohan/{{ item }}.git' - dest: "{{ WORKDIR }}/{{ item }}" - with_items: "{{ BOOTSTRAP_REPOS }}" - -- name: Create backup directory - ansible.builtin.file: - path: "{{ WORKDIR }}/backup" - state: directory - -- name: Archive Git repository - ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/{{ item }}.tar.gz -C {{ WORKDIR }} {{ item }}" - with_items: "{{ BOOTSTRAP_REPOS }}" - -- name: Copy secrets in backup directory - ansible.builtin.copy: - src: "{{ WORKDIR }}/secrets.tar.gz.enc" - dest: "{{ WORKDIR }}/backup/secrets.tar.gz.enc" - -thon3-swiftclient is a requirement of duplicity -- name: Install python3-swiftclient - ansible.builtin.package: - name: python3-swiftclient - state: present - -- name: Backup with duplicity - ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch '{{ WORKDIR }}/backup' swift://{{ lookup('env','BACKUP_WORKFLOW') }}" - environment: "{{ DUPLICITY_ENVIRONMENT }}" - -- name: Clean old duplicity backups - ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}" - environment: "{{ DUPLICITY_ENVIRONMENT }}" diff --git a/tasks/down.yml b/tasks/down.yml index 9a09e7e..3f5af0f 100755 --- a/tasks/down.yml +++ b/tasks/down.yml @@ -1,4 +1,26 @@ --- +#- name: Tear down existing services +# community.docker.docker_compose: +# project_src: /root/docker-duplicity-stack +# docker_host: tcp://127.0.0.1:2375 +# state: absent + +- name: Stop and remove duplicity container + community.docker.docker_container: + name: duplicity + state: absent + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + become: true + +- name: Remove docker-duplicity-stack directory + ansible.builtin.file: + path: "/home/{{ user }}/repository/docker-duplicity-stack" + state: absent + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + become: true + - name: unmount /mnt/cloud ansible.posix.mount: path: /mnt/cloud diff --git a/tasks/start.yml b/tasks/start.yml index bfe6007..6f769ce 100755 --- a/tasks/start.yml +++ b/tasks/start.yml @@ -19,12 +19,6 @@ state: directory mode: '0700' -- name: Copy SSH config - ansible.builtin.copy: - src: config - dest: "/root/.ssh" - mode: '0644' - - name: Extract from secrets.tar.gz.enc shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in /root/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 2 {{ item.name }}" with_items: @@ -37,10 +31,37 @@ - name: secrets/bootstrap/openrc.sh dir: /root -- name: Change secret files permissions +- name: Change SSH private key permissions ansible.builtin.file: - path: "{{ item }}" + path: /root/.ssh/id_rsa mode: '0400' + +- name: Remove docker-duplicity-stack directory + ansible.builtin.file: + path: "/home/{{ user }}/repository/docker-duplicity-stack" + state: absent + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + become: true + +- name: Clone docker-duplicity-stack repo + ansible.builtin.git: + repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity-stack.git' + dest: "/home/{{ user }}/repository/docker-duplicity-stack" + clone: yes + force: true + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + become: true + +- name: Copy files + ansible.builtin.copy: + src: "{{ item }}" + dest: "/home/{{ user }}/repository/docker-duplicity-stack" + mode: '0400' + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + become: true with_items: - /root/mail_credentials.json - /root/.ssh/id_rsa @@ -119,6 +140,73 @@ name: git state: present +- name: Get docker-duplicity repo's last commit + ansible.builtin.git: + repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git' + clone: no + update: no + version: master + register: git + +- name: Set fact tag + set_fact: + tag: "{{ git.after[0:10] }}" + +- name: Search for image + community.docker.docker_image: + name: duplicity + tag: "{{ tag }}" + source: local + docker_host: tcp://127.0.0.1:2375 + register: local_duplicity_image + failed_when: false + +- name: Create image build directory + ansible.builtin.file: + path: "/home/{{ user }}/build_docker-duplicity" + state: directory + when: + - local_duplicity_image.msg is defined + - '"Cannot find the image" in local_duplicity_image.msg' + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + +- name: Clone docker-duplicity repo + ansible.builtin.git: + repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git' + dest: "/home/{{ user }}/build_docker-duplicity" + clone: yes + version: master + when: + - local_duplicity_image.msg is defined + - '"Cannot find the image" in local_duplicity_image.msg' + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + +- name: Build duplicity image + community.docker.docker_image: + name: duplicity + tag: "{{ tag }}" + build: + path: "/home/{{ user }}/build_docker-duplicity" + source: build + when: + - local_duplicity_image.msg is defined + - '"Cannot find the image" in local_duplicity_image.msg' + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + become: true + +- name: Remove image build directory + ansible.builtin.file: + path: "/home/{{ user }}/build_docker-duplicity" + state: absent + when: + - local_duplicity_image.msg is defined + - '"Cannot find the image" in local_duplicity_image.msg' + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + - name: Install jsondiff from pip ansible.builtin.pip: name: jsondiff @@ -126,3 +214,65 @@ delegate_to: 172.17.0.1 become: true +- name: Clone duplicity_playbooks repo + ansible.builtin.git: + repo: 'https://{{ GIT_SERVER }}/yohan/duplicity_playbooks.git' + dest: "/home/{{ user }}/repository/duplicity_playbooks_temp" + clone: yes + version: master + force: true + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + +- name: Start duplicity container + community.docker.docker_container: + name: duplicity + image: "duplicity:{{ tag }}" + entrypoint: + - "ansible-playbook" + - "/root/duplicity_playbooks/{{ lookup('env','BACKUP_WORKFLOW') }}.yml" + output_logs: true + detach: false + network_mode: host + working_dir: "/home/{{ user }}/repository/docker-duplicity-stack" + volumes: + - /mnt/volumes:/mnt/volumes:z + - /mnt/cloud:/mnt/cloud:z + - /home/{{ user }}/repository/docker-duplicity-stack/backup_scripts:/mnt/scripts:z + - /home/{{ user }}/repository/docker-duplicity-stack/sendmail.py:/root/sendmail.py:z + - /home/{{ user }}/repository/docker-duplicity-stack/mail_credentials.json:/root/mail_credentials.json:z + - /home/{{ user }}/repository/docker-duplicity-stack/id_rsa:/root/.ssh/id_rsa:Z + - /home/{{ user }}/repository/docker-duplicity-stack/config:/root/.ssh/config:Z + - /home/{{ user }}/repository/duplicity_playbooks_temp:/root/duplicity_playbooks:Z + env: + OS_AUTH_URL: "{{ OS_AUTH_URL }}" + OS_IDENTITY_API_VERSION: "{{ OS_IDENTITY_API_VERSION }}" + OS_USER_DOMAIN_NAME: "{{ OS_USER_DOMAIN_NAME }}" + OS_PROJECT_DOMAIN_NAME: "{{ OS_PROJECT_DOMAIN_NAME }}" + OS_TENANT_ID: "{{ OS_TENANT_ID }}" + OS_TENANT_NAME: "{{ OS_TENANT_NAME }}" + OS_USERNAME: "{{ OS_USERNAME }}" + OS_PASSWORD: "{{ OS_PASSWORD }}" + OS_REGION_NAME: "{{ OS_SWIFT_REGION_NAME }}" + KEY: "{{ lookup('env','KEY') }}" + DOC_KEY: "{{ lookup('env','DOC_KEY') }}" + SECRETS_ARCHIVE_PASSPHRASE: "{{ lookup('env','SECRETS_ARCHIVE_PASSPHRASE') }}" + DUPLICITY_PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}" + BACKUP_WORKFLOW: "{{ lookup('env','BACKUP_WORKFLOW') }}" + remote_user: "{{ user }}" + delegate_to: 172.17.0.1 + become: true + register: container_output + +- debug: + msg: "{{ container_output.container.Output.split('\n') }}" + +# docker_compose collection version will not work on Centos 7 +#- name: Start duplicity stack +# community.docker.docker_compose: +# project_src: /home/{{ user }}/repository/docker-duplicity-stack +# state: present +# remote_user: "{{ user }}" +# delegate_to: 172.17.0.1 +# become: true + diff --git a/vars/main.yml b/vars/main.yml index 936ca26..1974f17 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,6 +1,7 @@ user: centos CLOUD_SERVER: cloud.scimetis.net GIT_SERVER: git.scimetis.net +OS_SWIFT_REGION_NAME: GRA WORKDIR: /mnt/volumes/tmp_duplicity_workdir/data ARCHIVE_DIR: /mnt/volumes/duplicity_cache/data @@ -13,17 +14,15 @@ BOOTSTRAP_REPOS: - docker-mysql - systemd-mount-cinder-volume -BOOTSTRAP_REQUIRED_VARS: - - SWIFT_AUTHURL - - SWIFT_AUTHVERSION - - SWIFT_TENANTNAME - - SWIFT_USERNAME - - SWIFT_PASSWORD - - SWIFT_REGION_NAME - - PASSPHRASE - BOOTSTRAP_REQUIRED_ENV_VARS: + - OS_AUTH_URL + - OS_IDENTITY_API_VERSION + - OS_TENANT_NAME + - OS_USERNAME + - OS_PASSWORD + - OS_REGION_NAME - SECRETS_ARCHIVE_PASSPHRASE + - DUPLICITY_PASSPHRASE - KEY - DOC_KEY - BACKUP_WORKFLOW @@ -39,16 +38,14 @@ BACKUP_OVH1_VOLUMES: - scuttle_code - scuttle_php5-fpm_conf -BACKUP_OVH1_REQUIRED_VARS: - - SWIFT_AUTHURL - - SWIFT_AUTHVERSION - - SWIFT_TENANTNAME - - SWIFT_USERNAME - - SWIFT_PASSWORD - - SWIFT_REGION_NAME - - PASSPHRASE - BACKUP_OVH1_REQUIRED_ENV_VARS: + - OS_AUTH_URL + - OS_IDENTITY_API_VERSION + - OS_TENANT_NAME + - OS_USERNAME + - OS_PASSWORD + - OS_REGION_NAME + - DUPLICITY_PASSPHRASE - BACKUP_WORKFLOW PLAYBOOK_REQUIRED_ENV_VARS: @@ -58,3 +55,12 @@ PLAYBOOK_REQUIRED_ENV_VARS: - DUPLICITY_PASSPHRASE - BACKUP_WORKFLOW +DUPLICITY_ENVIRONMENT: + SWIFT_AUTHURL: "{{ lookup('env','OS_AUTH_URL') }}" + SWIFT_AUTHVERSION: "{{ lookup('env','OS_IDENTITY_API_VERSION') }}" + SWIFT_TENANTNAME: "{{ lookup('env','OS_TENANT_NAME') }}" + SWIFT_USERNAME: "{{ lookup('env','OS_USERNAME') }}" + SWIFT_PASSWORD: "{{ lookup('env','OS_PASSWORD') }}" + SWIFT_REGION_NAME: "{{ lookup('env','OS_REGION_NAME') }}" + PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}" +