Bonjour,
+#
Test +#
+# +# +# """ + part2 = MIMEText(html, 'html', 'utf-8') + if args.attachment is not None: + text_msg.attach(part2) + else: + msg.attach(part2) + +if args.attachment is not None: + msg.attach(text_msg) + with open(args.attachment, 'rb') as file: + attachment = MIMEApplication(file.read(), 'octet-stream') + attachment.add_header('Content-Disposition', 'attachment', + filename=os.path.basename(file.name)) + msg.attach(attachment) + +# print msg.as_string().encode('ascii') +print "sending" +s = smtplib.SMTP('ssl0.ovh.net', 587) +s.set_debuglevel(1) +s.login(user, password) + +# sendmail function takes 3 arguments: sender's address, recipient's address +# and message to send - here it is sent as one string. +s.sendmail(sender, receiver, msg.as_string().encode('ascii')) +s.quit() diff --git a/launch_ansible_container.sh b/launch_ansible_container.sh index aae0a1f..04ec5bc 100755 --- a/launch_ansible_container.sh +++ b/launch_ansible_container.sh @@ -6,4 +6,4 @@ SCRIPTPATH=$(dirname $SCRIPT) cd $SCRIPTPATH USER=$(whoami) -sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -v $SCRIPTPATH:/root/duplicity_playbooks -i ansible /root/duplicity_playbooks/launch_top_playbook.sh +sudo -E docker run --net=host --rm -e KEY -e DOC_KEY -e SECRETS_ARCHIVE_PASSPHRASE -e DUPLICITY_PASSPHRASE -e BACKUP_WORKFLOW -v $SCRIPTPATH:/root/duplicity_playbooks -v /mnt:/mnt:z -i duplicity:master /root/duplicity_playbooks/launch_top_playbook.sh diff --git a/playbook.yml b/playbook.yml index 84ce5f3..f2a34f5 100755 --- a/playbook.yml +++ b/playbook.yml @@ -7,5 +7,17 @@ - name: Include start.yml ansible.builtin.include_tasks: "tasks/start.yml" + - name: Include {{ lookup('env','BACKUP_WORKFLOW') }}.yml + ansible.builtin.include_tasks: "tasks/{{ lookup('env','BACKUP_WORKFLOW') }}.yml" + vars: + DUPLICITY_ENVIRONMENT: + SWIFT_AUTHURL: "{{ OS_AUTH_URL }}" + SWIFT_AUTHVERSION: "{{ OS_IDENTITY_API_VERSION }}" + SWIFT_TENANTNAME: "{{ OS_TENANT_NAME }}" + SWIFT_USERNAME: "{{ OS_USERNAME }}" + SWIFT_PASSWORD: "{{ OS_PASSWORD }}" + SWIFT_REGION_NAME: GRA + PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}" + - name: Include down.yml ansible.builtin.include_tasks: "tasks/down.yml" diff --git a/tasks/assert_var_not_empty.yml b/tasks/assert_var_not_empty.yml new file mode 100755 index 0000000..0e7dde3 --- /dev/null +++ b/tasks/assert_var_not_empty.yml @@ -0,0 +1,6 @@ +--- +- name: Assert {{ item }} variable is set + ansible.builtin.assert: + that: + - item | length > 0 + msg: "{{ item }} variable must be set" diff --git a/tasks/backup_ovh1.yml b/tasks/backup_ovh1.yml new file mode 100755 index 0000000..8df339f --- /dev/null +++ b/tasks/backup_ovh1.yml @@ -0,0 +1,43 @@ +--- +- name: Assert environment variable is not empty + ansible.builtin.include_tasks: "tasks/assert_env_var_not_empty.yml" + with_items: "{{ BACKUP_OVH1_REQUIRED_ENV_VARS }}" + +- name: Assert variable is not empty + ansible.builtin.include_tasks: "tasks/assert_var_not_empty.yml" + with_items: "{{ BACKUP_OVH1_REQUIRED_VARS }}" + +- name: Create backup directory + ansible.builtin.file: + path: "{{ WORKDIR }}/backup" + state: directory + +- name: Archive volumes + ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/{{ item }}.tar.gz -C /mnt/volumes {{ item }}" + with_items: "{{ BACKUP_OVH1_VOLUMES }}" + +- name: Find lastest MySQL DB dump + ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump-mysql_*" + register: MySQL_dump + +- name: Find lastest applications DB dump + ansible.builtin.shell: "ls -tr /mnt/volumes/mysql-server_dumps/data/mysql_dump_*" + register: DBs_dump + +- name: Archive DB dumps + ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/mysql-server_dumps.tar.gz -C /mnt/volumes {{ MySQL_dump.stdout_lines | last }} {{ DBs_dump.stdout_lines | last }}" + with_items: "{{ BACKUP_OVH1_VOLUMES }}" + +thon3-swiftclient is a requirement of duplicity +- name: Install python3-swiftclient + ansible.builtin.package: + name: python3-swiftclient + state: present + +- name: Backup with duplicity + ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch '{{ WORKDIR }}/backup' swift://{{ lookup('env','BACKUP_WORKFLOW') }}" + environment: "{{ DUPLICITY_ENVIRONMENT }}" + +- name: Clean old duplicity backups + ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}" + environment: "{{ DUPLICITY_ENVIRONMENT }}" diff --git a/tasks/bootstrap.yml b/tasks/bootstrap.yml new file mode 100755 index 0000000..30fb74a --- /dev/null +++ b/tasks/bootstrap.yml @@ -0,0 +1,128 @@ +--- +- name: Assert environment variable is not empty + ansible.builtin.include_tasks: "tasks/assert_env_var_not_empty.yml" + with_items: "{{ BOOTSTRAP_REQUIRED_ENV_VARS }}" + +- name: Assert variable is not empty + ansible.builtin.include_tasks: "tasks/assert_var_not_empty.yml" + with_items: "{{ BOOTSTRAP_REQUIRED_VARS }}" + +- name: Download secrets.tar.gz.enc + ansible.builtin.get_url: + url: "https://{{ CLOUD_SERVER }}/s/{{ lookup('env','KEY') }}/download?path=%2F&files=secrets.tar.gz.enc" + dest: "{{ WORKDIR }}/secrets.tar.gz.enc" + +- name: Install openssh-client + ansible.builtin.package: + name: openssh-client + state: present + +- name: Create /root/.ssh directory + ansible.builtin.file: + path: /root/.ssh + state: directory + mode: '0700' + +- name: Extract from secrets.tar.gz.enc + shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in {{ WORKDIR }}/secrets.tar.gz.enc | tar -zxv -C {{ WORKDIR }}" + +- name: Change SSH private key permissions + ansible.builtin.file: + path: /root/.ssh/id_rsa + mode: '0400' + +- name: Retrieve documentation + ansible.builtin.get_url: + url: "https://{{ CLOUD_SERVER }}/s/{{ lookup('env','DOC_KEY') }}/download" + dest: "{{ WORKDIR }}/Documentation.md" + +- name: Copy new documentation + ansible.builtin.copy: + src: "{{ WORKDIR }}/Documentation.md" + dest: "{{ WORKDIR }}/secrets/bootstrap/Documentation.md" + register: copy_output + +- name: Create secrets.tar.gz.enc + shell: "tar -czvpf - -C {{ WORKDIR }} secrets | openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -salt -out {{ WORKDIR }}/secrets.tar.gz.enc" + when: copy_output is changed + +- name: Copy mail content + ansible.builtin.copy: + content: "Secrets archive has changed. New file attached." + dest: "{{ WORKDIR }}/mail" + when: copy_output is changed + +- name: Install python2 + ansible.builtin.package: + name: python2 + state: present + +- name: Send mail with new secrets + ansible.builtin.command: "files/sendmail.py -a {{ WORKDIR }}/secrets.tar.gz.enc {{ WORKDIR }}/mail /root/mail_credentials.json" + when: copy_output is changed + +- name: Copy new secrets in Nextcloud share + ansible.builtin.copy: + src: "{{ WORKDIR }}/secrets.tar.gz.enc" + dest: /mnt/cloud/Passwords/secrets.tar.gz.enc + when: copy_output is changed + +- name: Create /mnt/archives_critiques/secrets directory on serveur-appart + ansible.builtin.file: + path: /mnt/archives_critiques/secrets + state: directory + owner: yohan + group: yohan + remote_user: yohan + vars: + ansible_ssh_port: 2224 + delegate_to: chez-yohan.scimetis.net + become: true + +- name: Get checksum of secrets.tar.gz.enc + ansible.builtin.stat: + path: "{{ WORKDIR }}/secrets.tar.gz.enc" + register: stats_output + +- name: Copy new secrets on serveur-appart + ansible.builtin.copy: + src: "{{ WORKDIR }}/secrets.tar.gz.enc" + dest: "/mnt/archives_critiques/secrets/secrets.tar.gz.enc-{{ stats_output.stat.checksum }}" + remote_user: yohan + vars: + ansible_ssh_port: 2224 + delegate_to: chez-yohan.scimetis.net + +- name: Clone repo + ansible.builtin.git: + repo: 'https://{{ GIT_SERVER }}/yohan/{{ item }}.git' + dest: "{{ WORKDIR }}/{{ item }}" + with_items: "{{ BOOTSTRAP_REPOS }}" + +- name: Create backup directory + ansible.builtin.file: + path: "{{ WORKDIR }}/backup" + state: directory + +- name: Archive Git repository + ansible.builtin.command: "tar -czf {{ WORKDIR }}/backup/{{ item }}.tar.gz -C {{ WORKDIR }} {{ item }}" + with_items: "{{ BOOTSTRAP_REPOS }}" + +- name: Copy secrets in backup directory + ansible.builtin.copy: + src: "{{ WORKDIR }}/secrets.tar.gz.enc" + dest: "{{ WORKDIR }}/backup/secrets.tar.gz.enc" + +thon3-swiftclient is a requirement of duplicity +- name: Install python3-swiftclient + ansible.builtin.package: + name: python3-swiftclient + state: present + +- name: Backup with duplicity + ansible.builtin.command: "duplicity --num-retries 3 --full-if-older-than 1M --progress --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch '{{ WORKDIR }}/backup' swift://{{ lookup('env','BACKUP_WORKFLOW') }}" + environment: "{{ DUPLICITY_ENVIRONMENT }}" + +- name: Clean old duplicity backups + ansible.builtin.command: "duplicity remove-older-than 2M --archive-dir {{ ARCHIVE_DIR }} --name {{ lookup('env','BACKUP_WORKFLOW') }} --allow-source-mismatch --force swift://{{ lookup('env','BACKUP_WORKFLOW') }}" + environment: "{{ DUPLICITY_ENVIRONMENT }}" diff --git a/tasks/down.yml b/tasks/down.yml index 3f5af0f..9a09e7e 100755 --- a/tasks/down.yml +++ b/tasks/down.yml @@ -1,26 +1,4 @@ --- -#- name: Tear down existing services -# community.docker.docker_compose: -# project_src: /root/docker-duplicity-stack -# docker_host: tcp://127.0.0.1:2375 -# state: absent - -- name: Stop and remove duplicity container - community.docker.docker_container: - name: duplicity - state: absent - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - become: true - -- name: Remove docker-duplicity-stack directory - ansible.builtin.file: - path: "/home/{{ user }}/repository/docker-duplicity-stack" - state: absent - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - become: true - - name: unmount /mnt/cloud ansible.posix.mount: path: /mnt/cloud diff --git a/tasks/start.yml b/tasks/start.yml index 6f769ce..bfe6007 100755 --- a/tasks/start.yml +++ b/tasks/start.yml @@ -19,6 +19,12 @@ state: directory mode: '0700' +- name: Copy SSH config + ansible.builtin.copy: + src: config + dest: "/root/.ssh" + mode: '0644' + - name: Extract from secrets.tar.gz.enc shell: "openssl enc -aes-256-cbc -md md5 -pass env:SECRETS_ARCHIVE_PASSPHRASE -d -in /root/secrets.tar.gz.enc | tar -zxv -C {{ item.dir }} --strip 2 {{ item.name }}" with_items: @@ -31,37 +37,10 @@ - name: secrets/bootstrap/openrc.sh dir: /root -- name: Change SSH private key permissions +- name: Change secret files permissions ansible.builtin.file: - path: /root/.ssh/id_rsa + path: "{{ item }}" mode: '0400' - -- name: Remove docker-duplicity-stack directory - ansible.builtin.file: - path: "/home/{{ user }}/repository/docker-duplicity-stack" - state: absent - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - become: true - -- name: Clone docker-duplicity-stack repo - ansible.builtin.git: - repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity-stack.git' - dest: "/home/{{ user }}/repository/docker-duplicity-stack" - clone: yes - force: true - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - become: true - -- name: Copy files - ansible.builtin.copy: - src: "{{ item }}" - dest: "/home/{{ user }}/repository/docker-duplicity-stack" - mode: '0400' - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - become: true with_items: - /root/mail_credentials.json - /root/.ssh/id_rsa @@ -140,73 +119,6 @@ name: git state: present -- name: Get docker-duplicity repo's last commit - ansible.builtin.git: - repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git' - clone: no - update: no - version: master - register: git - -- name: Set fact tag - set_fact: - tag: "{{ git.after[0:10] }}" - -- name: Search for image - community.docker.docker_image: - name: duplicity - tag: "{{ tag }}" - source: local - docker_host: tcp://127.0.0.1:2375 - register: local_duplicity_image - failed_when: false - -- name: Create image build directory - ansible.builtin.file: - path: "/home/{{ user }}/build_docker-duplicity" - state: directory - when: - - local_duplicity_image.msg is defined - - '"Cannot find the image" in local_duplicity_image.msg' - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - -- name: Clone docker-duplicity repo - ansible.builtin.git: - repo: 'https://{{ GIT_SERVER }}/yohan/docker-duplicity.git' - dest: "/home/{{ user }}/build_docker-duplicity" - clone: yes - version: master - when: - - local_duplicity_image.msg is defined - - '"Cannot find the image" in local_duplicity_image.msg' - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - -- name: Build duplicity image - community.docker.docker_image: - name: duplicity - tag: "{{ tag }}" - build: - path: "/home/{{ user }}/build_docker-duplicity" - source: build - when: - - local_duplicity_image.msg is defined - - '"Cannot find the image" in local_duplicity_image.msg' - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - become: true - -- name: Remove image build directory - ansible.builtin.file: - path: "/home/{{ user }}/build_docker-duplicity" - state: absent - when: - - local_duplicity_image.msg is defined - - '"Cannot find the image" in local_duplicity_image.msg' - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - - name: Install jsondiff from pip ansible.builtin.pip: name: jsondiff @@ -214,65 +126,3 @@ delegate_to: 172.17.0.1 become: true -- name: Clone duplicity_playbooks repo - ansible.builtin.git: - repo: 'https://{{ GIT_SERVER }}/yohan/duplicity_playbooks.git' - dest: "/home/{{ user }}/repository/duplicity_playbooks_temp" - clone: yes - version: master - force: true - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - -- name: Start duplicity container - community.docker.docker_container: - name: duplicity - image: "duplicity:{{ tag }}" - entrypoint: - - "ansible-playbook" - - "/root/duplicity_playbooks/{{ lookup('env','BACKUP_WORKFLOW') }}.yml" - output_logs: true - detach: false - network_mode: host - working_dir: "/home/{{ user }}/repository/docker-duplicity-stack" - volumes: - - /mnt/volumes:/mnt/volumes:z - - /mnt/cloud:/mnt/cloud:z - - /home/{{ user }}/repository/docker-duplicity-stack/backup_scripts:/mnt/scripts:z - - /home/{{ user }}/repository/docker-duplicity-stack/sendmail.py:/root/sendmail.py:z - - /home/{{ user }}/repository/docker-duplicity-stack/mail_credentials.json:/root/mail_credentials.json:z - - /home/{{ user }}/repository/docker-duplicity-stack/id_rsa:/root/.ssh/id_rsa:Z - - /home/{{ user }}/repository/docker-duplicity-stack/config:/root/.ssh/config:Z - - /home/{{ user }}/repository/duplicity_playbooks_temp:/root/duplicity_playbooks:Z - env: - OS_AUTH_URL: "{{ OS_AUTH_URL }}" - OS_IDENTITY_API_VERSION: "{{ OS_IDENTITY_API_VERSION }}" - OS_USER_DOMAIN_NAME: "{{ OS_USER_DOMAIN_NAME }}" - OS_PROJECT_DOMAIN_NAME: "{{ OS_PROJECT_DOMAIN_NAME }}" - OS_TENANT_ID: "{{ OS_TENANT_ID }}" - OS_TENANT_NAME: "{{ OS_TENANT_NAME }}" - OS_USERNAME: "{{ OS_USERNAME }}" - OS_PASSWORD: "{{ OS_PASSWORD }}" - OS_REGION_NAME: "{{ OS_SWIFT_REGION_NAME }}" - KEY: "{{ lookup('env','KEY') }}" - DOC_KEY: "{{ lookup('env','DOC_KEY') }}" - SECRETS_ARCHIVE_PASSPHRASE: "{{ lookup('env','SECRETS_ARCHIVE_PASSPHRASE') }}" - DUPLICITY_PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}" - BACKUP_WORKFLOW: "{{ lookup('env','BACKUP_WORKFLOW') }}" - remote_user: "{{ user }}" - delegate_to: 172.17.0.1 - become: true - register: container_output - -- debug: - msg: "{{ container_output.container.Output.split('\n') }}" - -# docker_compose collection version will not work on Centos 7 -#- name: Start duplicity stack -# community.docker.docker_compose: -# project_src: /home/{{ user }}/repository/docker-duplicity-stack -# state: present -# remote_user: "{{ user }}" -# delegate_to: 172.17.0.1 -# become: true - diff --git a/vars/main.yml b/vars/main.yml index 1974f17..936ca26 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,7 +1,6 @@ user: centos CLOUD_SERVER: cloud.scimetis.net GIT_SERVER: git.scimetis.net -OS_SWIFT_REGION_NAME: GRA WORKDIR: /mnt/volumes/tmp_duplicity_workdir/data ARCHIVE_DIR: /mnt/volumes/duplicity_cache/data @@ -14,15 +13,17 @@ BOOTSTRAP_REPOS: - docker-mysql - systemd-mount-cinder-volume +BOOTSTRAP_REQUIRED_VARS: + - SWIFT_AUTHURL + - SWIFT_AUTHVERSION + - SWIFT_TENANTNAME + - SWIFT_USERNAME + - SWIFT_PASSWORD + - SWIFT_REGION_NAME + - PASSPHRASE + BOOTSTRAP_REQUIRED_ENV_VARS: - - OS_AUTH_URL - - OS_IDENTITY_API_VERSION - - OS_TENANT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_REGION_NAME - SECRETS_ARCHIVE_PASSPHRASE - - DUPLICITY_PASSPHRASE - KEY - DOC_KEY - BACKUP_WORKFLOW @@ -38,14 +39,16 @@ BACKUP_OVH1_VOLUMES: - scuttle_code - scuttle_php5-fpm_conf +BACKUP_OVH1_REQUIRED_VARS: + - SWIFT_AUTHURL + - SWIFT_AUTHVERSION + - SWIFT_TENANTNAME + - SWIFT_USERNAME + - SWIFT_PASSWORD + - SWIFT_REGION_NAME + - PASSPHRASE + BACKUP_OVH1_REQUIRED_ENV_VARS: - - OS_AUTH_URL - - OS_IDENTITY_API_VERSION - - OS_TENANT_NAME - - OS_USERNAME - - OS_PASSWORD - - OS_REGION_NAME - - DUPLICITY_PASSPHRASE - BACKUP_WORKFLOW PLAYBOOK_REQUIRED_ENV_VARS: @@ -55,12 +58,3 @@ PLAYBOOK_REQUIRED_ENV_VARS: - DUPLICITY_PASSPHRASE - BACKUP_WORKFLOW -DUPLICITY_ENVIRONMENT: - SWIFT_AUTHURL: "{{ lookup('env','OS_AUTH_URL') }}" - SWIFT_AUTHVERSION: "{{ lookup('env','OS_IDENTITY_API_VERSION') }}" - SWIFT_TENANTNAME: "{{ lookup('env','OS_TENANT_NAME') }}" - SWIFT_USERNAME: "{{ lookup('env','OS_USERNAME') }}" - SWIFT_PASSWORD: "{{ lookup('env','OS_PASSWORD') }}" - SWIFT_REGION_NAME: "{{ lookup('env','OS_REGION_NAME') }}" - PASSPHRASE: "{{ lookup('env','DUPLICITY_PASSPHRASE') }}" -