diff --git a/.gitignore b/.gitignore index a5e5076..76b3ec6 100644 --- a/.gitignore +++ b/.gitignore @@ -35,9 +35,10 @@ nosetests.xml .project .pydevproject -ansible.cfg *.pem *.torrent murder_dist.tar.gz stage murder/ +CMDB_share_credentials +vcsim_vars.yml diff --git a/README.md b/README.md index 599816b..5993e93 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,18 @@ ansible-lssd ============ -Large scale server deploys using BitTorrent and the BitTornado library by [Murder](https://github.com/lg/murder). -Mechanism is the same as Murder. -For an intro video, see: [Twitter - Murder Bittorrent Deploy System](http://vimeo.com/11280885) +Large scale server deploys using BitTorrent. aria2c, NodeJS Bittorrent-tracker and mktorrent are used. DESCRIPTION ----------- -ansible-lssd is a method of using Bittorrent (Powered by [Murder](https://github.com/lg/murder)) to distribute files to a large amount of servers within a production environment. +ansible-lssd is a method of using Bittorrent to distribute files to a large amount of servers within a production environment. ansible-lssd operates as a playbook of [Ansible](https://github.com/ansible/ansible). These playbooks require Ansible 1.4. -These playbooks were tested on CentOS 5.x so we recommend that you use CentOS to test these modules. +These playbooks were tested on CentOS 7.x (ansible 2.6) with peers on Centos 7.x and Centos 6.x. Advantage --------- @@ -28,12 +26,8 @@ There is the following advantage as compared with Murder. Installation of software required minimum ----------------------------------------- -`sudo pip install ansible` : - ```bash: - # install pip (Python package manager) and ansible - $ sudo easy_install pip - $ sudo pip install ansible - ``` +From EPEL: +# yum -y install ansible HOW IT WORKS @@ -47,15 +41,9 @@ CONFIGURATION You define `tracker`, `seeder` and `peer` server to inventory (./production) file. -All involved servers must have python installed and the related murder -support files (BitTornado, Murder lib, screen, pigz and etc.). To upload the support files to the tracker, seeder, and peers, run: - ```bash: - $ ansible-playbook -i prodction setup.yml - ``` - -By default, these will go in `/usr/local/murder` in your apps deploy directory. -Override this by setting the variable `remote_murder_path`. +All involved servers must have python installed. +The file group_vars/all contains the configuration. MANUAL USAGE ------------ @@ -100,7 +88,7 @@ Then manually run the ansible playbooks: 1. Start the tracker: ```bash: - $ ansible-playbook -i prodction start_tracker.yml + $ ansible-playbook -i production start_tracker.yml ``` 2. Create a torrent from a directory of files on the seeder, and start seeding: @@ -110,22 +98,22 @@ Then manually run the ansible playbooks: $ scp -r ./builds 10.0.0.1:~/builds # create torrent file - $ ansible-playbook -i prodction create_torrent.yml + $ ansible-playbook -i production create_torrent.yml # start seeding - $ ansible-playbook -i prodction start_seeder.yml + $ ansible-playbook -i production start_seeder.yml ``` 3. Distribute the torrent to all peers: ```bash: - $ ansible-playbook -i prodction deploy.yml -f 1000 + $ ansible-playbook -i production deploy.yml -f 1000 ``` 4. Stop the seeder and tracker: ```bash: - $ ansible-playbook -i prodction stop_seeder_and_tracker.yml + $ ansible-playbook -i production stop_seeder_and_tracker.yml ``` When this finishes, all peers will have the files in /opt/hoge/Deploy1 @@ -138,11 +126,9 @@ MAIN PLAYBOOKS REFERENCE * Create torrent file on seeder node. * `deploy.yml:` * Deploy files on peer nodes. -* `rm_tgz.yml:` - * Delete the file deployment of targz in all peer node. * `setup.yml:` * Install the software required for each node. -* `site.yml:` +* `full_deploy.yml:` * Run all Playbook deploy from the setup. * `start_seeder.yml:` * start seeding. @@ -154,6 +140,10 @@ MAIN PLAYBOOKS REFERENCE * stop seeding and tracker. +NOTES +------------------------ +You may hit the request length limit (fixed in newer aria2 releases) : +https://github.com/aria2/aria2/commit/e220c5384961f9261f19c28cd0b85f76f06d8993#diff-03671aebef9174610c96db97917b960a - +Workaround is using "-l 22" as mktorrent option to make less pieces. diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..2e5fffb --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +forks = 150 +host_key_checking = False +callback_whitelist = profile_tasks diff --git a/bittorrent-tracker.tar.gz b/bittorrent-tracker.tar.gz new file mode 100755 index 0000000..1202a3f Binary files /dev/null and b/bittorrent-tracker.tar.gz differ diff --git a/create_inventory.sh b/create_inventory.sh new file mode 100755 index 0000000..b6badc8 --- /dev/null +++ b/create_inventory.sh @@ -0,0 +1,23 @@ +#!/usr/bin/python + +#relays={"HLZURT1EXE3M": {"Bat": "Bat Diderot", "Salle": "Salle 1"}, "HLZURT1EXE3N": {"Bat": "D'alembert", "Salle": "Salle 2"}} +relays={"HLZURT1EXE3M": {"Bat": "Bat C", "Salle": "Salle C31"}} +inventory_file="associations_vm-location" + +for relay in relays.keys(): + with open("inventory_" + relay, "w") as f: + f.write("# ansible host\n") + f.write("[ansible_host]\n") + f.write("localhost ansible_connection=local\n\n") + f.write("# tracker node\n") + f.write("[tracker]\n") + f.write(relay.lower() + " node_type=tracker\n\n") + f.write("# seeder node\n") + f.write("[seeder]\n") + f.write(relay.lower() + " node_type=seeder\n\n") + f.write("# peer nodes\n") + f.write("[peer]\n") + with open(inventory_file, "r") as inventory: + for line in inventory: + if relays[relay]["Bat"] in line and relays[relay]["Salle"] in line: + f.write(line.split(';')[0]+"\n") diff --git a/create_inventory.yml b/create_inventory.yml new file mode 100644 index 0000000..57be5f4 --- /dev/null +++ b/create_inventory.yml @@ -0,0 +1,104 @@ +- name: create_inventory + gather_facts: no + vars_files: + - vcsim_vars.yml + - vm_list.yml + hosts: localhost + + tasks: +# - name: Gather VMware guest facts +# vmware_guest_facts: +# hostname: "{{ vcenter_server }}" +# username: "{{ vcenter_user }}" +# password: "{{ vcenter_pass }}" +# datacenter: +# name: "{{ item }}" +# validate_certs: no +# register: vm_result +# with_items: "{{ vm_list }}" + + - name: Gather VMware guest facts + vmware_guest_facts: + hostname: "{{ item[1] }}" + username: "{{ vcenter_user }}" + password: "{{ vcenter_pass }}" + datacenter: + name: "{{ item[0] }}" + validate_certs: no + register: vm_result + ignore_errors: true + with_nested: + - "{{ vm_list }}" + - "{{ vcenter_servers }}" + + - name: remove output file + file: name=./associations_vm-esx state=absent + + - name: make output file + file: name=./associations_vm-esx state=touch + + - name: Get ESXs of VMs + vars: + s_query: "[?instance.hw_name == '{{ item }}'].instance.hw_esxi_host" + with_items: "{{ vm_list }}" + lineinfile: dest=./associations_vm-esx line="{{ item }};{{ vm_result.results | json_query(s_query) | join }}" + + - name: check results + shell: "[ $(grep {{ item }} ./associations_vm-esx| awk -F';' '{print $2}') != '' ] || { echo 'Could not find ESX for VM : {{ item }}'; exit 1; }" + with_items: "{{ vm_list }}" + + - name: Get ESXs of VMs2 + set_fact: + esx_list: "{{ esx_list|default([]) + [ vm_result.results | json_query(s_query) | join ] }}" + vars: + s_query: "[?instance.hw_name == '{{ item }}'].instance.hw_esxi_host" + with_items: "{{ vm_list }}" + + - name: remove output file + file: name=./associations_vm-location state=absent + + - name: make output file + file: name=./associations_vm-location state=touch + + - name: create mount directory + file: name=/mnt/CMDB state=directory + become: yes + + - name: mount CMDB share + shell: mountpoint -q /mnt/CMDB || mount -t cifs //CPMUPD2APPVQ.ZRES.ZTECH/mup10cmdb_s/Envoi/PUBLICATION /mnt/CMDB -o credentials={{playbook_dir}}/CMDB_share_credentials,domain=ZGIE + become: yes + + - name: find CMDB file + shell: ls -t /mnt/CMDB/MUP10CMDBH_*LISTE_CS.CSV|head -n1 + register: CMDB_file + + - name: store CMDB file path + set_fact: + CMDB_file_path: "{{ CMDB_file.stdout }}" + + - name: retrieve ESXs' room + shell: grep -i "{{ item.split('.')[0] }}" {{ CMDB_file_path }}|awk -F',' '{print $14}'|tr -d '\t\n' + register: esx_locations + with_items: "{{ esx_list }}" + + - name: umount CMDB share + shell: mountpoint -q /mnt/CMDB && umount /mnt/CMDB + become: yes + + - name: store location + set_fact: + location: "{{ esx_locations.results | map(attribute='stdout') | list }}" + + - name: store associations + set_fact: + associations: "{{ (vm_list|zip([';'] * vm_list|length)|map('join')|list) | zip(location) | list | map('join') | list }}" + + - name: save associations in file + #debug: + # msg: "{{ item }}" + lineinfile: dest=./associations_vm-location line="{{ item }}" + with_items: "{{ associations }}" + + - name: create inventories + command: ./create_inventory.sh + diff --git a/site.yml b/full_deploy.yml similarity index 80% rename from site.yml rename to full_deploy.yml index 6b16e3b..60c72b3 100644 --- a/site.yml +++ b/full_deploy.yml @@ -1,11 +1,9 @@ --- -- include: setup.yml - include: stop_seeder_and_tracker.yml - include: stop_all_peers.yml -- include: rm_tgz.yml +- include: setup.yml - include: start_tracker.yml -- include: create_torrent.yml +#- include: create_torrent.yml - include: start_seeder.yml - include: deploy.yml - include: stop_seeder_and_tracker.yml - diff --git a/group_vars/all b/group_vars/all index ee262ab..eaaa91f 100644 --- a/group_vars/all +++ b/group_vars/all @@ -1,30 +1,36 @@ # The global variable file # deploy tag -tag: deploy_test2 +tag: archive_clonage + +# ADABO +seeder_ADABO: ZUR10 +peer_ADABO: ZUR30 # path -seeder_files_path: ~/builds -destination_path: /product/hoge +seeder_files_path: /data/{{ seeder_ADABO | upper }}/Sas/clonage/Envoi +destination_path: /opt/{{ peer_ADABO | upper }}/Logiciel/BDIG/aem-install/crx-quickstart # path permission -destination_owner: root -destination_group: root -destination_mod: 755 +destination_owner: cdap{{ peer_ADABO | lower }}bdig +destination_group: go_{{ peer_ADABO | lower }}_bdig +destination_mod: "0755" # default tracker port -tracker_port: 8998 +tracker_port: 6969 # tar options tar_options: --exclude ".git*" --exclude ".svn*" # options -no_tag_directory: false +no_tag_directory: true unsafe_please_delete: false +only_deploy_archive: true +remove_existing: false # system path -torrent_path: . -remote_murder_path: /usr/local/murder -ansible_cache_path: /var/cache/ansible -default_temp_path: /tmp -murder_dist_tgz_file: ./murder_dist.tar.gz +src_path: "{{ playbook_dir }}" +remote_bittorrent_tracker_path: /usr/local/bittorrent-tracker +ansible_cache_path: /var/cache/ansible +default_temp_path: /tmp +bittorrent_tracker_tgz_file: bittorrent-tracker.tar.gz diff --git a/inventory_HLZURT1EXE3M b/inventory_HLZURT1EXE3M new file mode 100644 index 0000000..1f8ea62 --- /dev/null +++ b/inventory_HLZURT1EXE3M @@ -0,0 +1,18 @@ +# ansible host +[ansible_host] +localhost ansible_connection=local + +# tracker node +[tracker] +hlzurt1exe3m node_type=tracker + +# seeder node +[seeder] +hlzurt1exe3m node_type=seeder + +# peer nodes +[peer] +HLZURT1EXE04 +HLZURT1EXE05 +HLZURT1EXE06 +HLZURT1EXE08 diff --git a/packages/aria2-1.16.4-1.el6.rf.x86_64.rpm b/packages/aria2-1.16.4-1.el6.rf.x86_64.rpm new file mode 100644 index 0000000..670170f Binary files /dev/null and b/packages/aria2-1.16.4-1.el6.rf.x86_64.rpm differ diff --git a/packages/aria2-1.18.10-2.el7.1.x86_64.rpm b/packages/aria2-1.18.10-2.el7.1.x86_64.rpm new file mode 100644 index 0000000..e22ba15 Binary files /dev/null and b/packages/aria2-1.18.10-2.el7.1.x86_64.rpm differ diff --git a/packages/nettle-2.2-1.el6.rf.x86_64.rpm b/packages/nettle-2.2-1.el6.rf.x86_64.rpm new file mode 100644 index 0000000..2201024 Binary files /dev/null and b/packages/nettle-2.2-1.el6.rf.x86_64.rpm differ diff --git a/packages/pigz-2.3.4-1.el6.x86_64.rpm b/packages/pigz-2.3.4-1.el6.x86_64.rpm new file mode 100644 index 0000000..95e9798 Binary files /dev/null and b/packages/pigz-2.3.4-1.el6.x86_64.rpm differ diff --git a/packages/pigz-2.3.4-1.el7.x86_64.rpm b/packages/pigz-2.3.4-1.el7.x86_64.rpm new file mode 100644 index 0000000..321591a Binary files /dev/null and b/packages/pigz-2.3.4-1.el7.x86_64.rpm differ diff --git a/rm_tgz.yml b/rm_tgz.yml deleted file mode 100644 index 9cf4609..0000000 --- a/rm_tgz.yml +++ /dev/null @@ -1,9 +0,0 @@ - - -- name: rm_tgz - gather_facts: no - hosts: peer - sudo: yes - roles: - - murder/rm_tgz - diff --git a/roles/admin/clone_murder_files/tasks/main.yml b/roles/admin/clone_murder_files/tasks/main.yml deleted file mode 100644 index dd7fcd4..0000000 --- a/roles/admin/clone_murder_files/tasks/main.yml +++ /dev/null @@ -1,14 +0,0 @@ - -- name: clone murder - git: repo=https://github.com/lg/murder.git dest=./murder - register: clone_status - tags: clone_murder_files - -- name: rm old tgz - shell: rm -f {{ murder_dist_tgz_file }} - when: clone_status.changed - -- name: tar -czf - shell: tar -czf {{ murder_dist_tgz_file }} -C ./murder/dist/ . creates={{ murder_dist_tgz_file }} - tags: clone_murder_files - diff --git a/roles/admin/deploy_bittorrent-tracker/tasks/main.yml b/roles/admin/deploy_bittorrent-tracker/tasks/main.yml new file mode 100644 index 0000000..bdc2e35 --- /dev/null +++ b/roles/admin/deploy_bittorrent-tracker/tasks/main.yml @@ -0,0 +1,31 @@ +# SCPs a compressed version of all files from ./dist (the python Bittorrent library and custom scripts) to all server. +# The entire directory is sent, regardless of the role of each individual server. +# The path on the server is specified by remote_bittorrent-tracker_path and will be cleared prior to transferring files over. + +- name: upload + copy: src={{ bittorrent_tracker_tgz_file }} dest={{ ansible_cache_path }}/bittorrent-tracker.tar.gz + tags: deploy_bittorrent-tracker + register: copy_status + +- name: clean up remote_bittorrent-tracker_path + shell: | + [ $(find {{ remote_bittorrent_tracker_path }}/* | wc -l ) -lt 1000 ] && + rm -rf {{ remote_bittorrent_tracker_path }} || + ( echo 'Cowardly refusing to remove files! Check the remote_bittorrent-tracker_path.' ; exit 1 ) + when: copy_status.changed + register: rm_status + tags: deploy_bittorrent-tracker + +- name: dir stat + stat: path={{ remote_bittorrent_tracker_path }}/node_modules + register: st + +- name: tar -xzf + shell: mkdir -p {{ remote_bittorrent_tracker_path }} && tar -xzf {{ ansible_cache_path }}/bittorrent-tracker.tar.gz -C {{ remote_bittorrent_tracker_path }} + when: st.stat.isdir is undefined + tags: deploy_bittorrent-tracker + +- name: install NodeJS + yum: name=nodejs state=latest + tags: deploy_bittorrent-tracker + diff --git a/roles/admin/distribute_files/tasks/main.yml b/roles/admin/distribute_files/tasks/main.yml deleted file mode 100644 index bb9012c..0000000 --- a/roles/admin/distribute_files/tasks/main.yml +++ /dev/null @@ -1,32 +0,0 @@ -# SCPs a compressed version of all files from ./dist (the python Bittorrent library and custom scripts) to all server. -# The entire directory is sent, regardless of the role of each individual server. -# The path on the server is specified by remote_murder_path and will be cleared prior to transferring files over. -- name: mkdir ansible_cache_path - file: path={{ ansible_cache_path }} state=directory owner=root group=root mode=0755 - tags: distribute_files - - -- name: upload - copy: src={{ murder_dist_tgz_file }} dest={{ ansible_cache_path }}/murder_dist.tar.gz - tags: distribute_files - register: copy_status - -- name: clean up remote_murder_path - shell: | - [ $(find {{ remote_murder_path }}/* | wc -l ) -lt 1000 ] && - rm -rf {{ remote_murder_path }} || - ( echo 'Cowardly refusing to remove files! Check the remote_murder_path.' ; exit 1 ) - when: copy_status.changed - register: rm_status - tags: distribute_files - -- name: dir stat - stat: path={{ remote_murder_path }}/BitTornado - register: st - -- name: tar -xzf - shell: mkdir -p {{ remote_murder_path }} && tar -xzf {{ ansible_cache_path }}/murder_dist.tar.gz -C {{ remote_murder_path }} - when: st.stat.isdir is undefined - tags: distribute_files - - diff --git a/roles/admin/pigz/tasks/.main.yml.swp b/roles/admin/pigz/tasks/.main.yml.swp new file mode 100644 index 0000000..95693fc Binary files /dev/null and b/roles/admin/pigz/tasks/.main.yml.swp differ diff --git a/roles/admin/pigz/tasks/main.yml b/roles/admin/pigz/tasks/main.yml index b59ae04..e20bf0c 100644 --- a/roles/admin/pigz/tasks/main.yml +++ b/roles/admin/pigz/tasks/main.yml @@ -1,5 +1,19 @@ +- name: copy pigz 6 + copy: + src: packages/pigz-2.3.4-1.el6.x86_64.rpm + dest: "{{ ansible_cache_path }}" + when: ansible_distribution_major_version == "6" -- name: yum install pigz - yum: name=pigz state=latest - tags: pigz +- name: copy pigz 7 + copy: + src: packages/pigz-2.3.4-1.el7.x86_64.rpm + dest: "{{ ansible_cache_path }}" + when: ansible_distribution_major_version == "7" + +- name: yum localinstall pigz + shell: yum -y localinstall "{{ ansible_cache_path }}"/pigz*.rpm --nogpgcheck + +#- name: yum install pigz +# yum: name=pigz state=latest +# tags: pigz diff --git a/roles/admin/prepare/tasks/main.yml b/roles/admin/prepare/tasks/main.yml new file mode 100644 index 0000000..27db78f --- /dev/null +++ b/roles/admin/prepare/tasks/main.yml @@ -0,0 +1,13 @@ +- name: check ansible_cache_path + stat: path={{ ansible_cache_path }} + register: test_path + tags: prepare + +- name: remove ansible_cache_path if not a directory + file: path={{ ansible_cache_path }} state=absent + when: test_path.stat is defined and (test_path.stat.isdir is not defined or not test_path.stat.isdir) + tags: prepare + +- name: mkdir ansible_cache_path + file: path={{ ansible_cache_path }} state=directory owner=root group=root mode=0755 + tags: prepare diff --git a/roles/admin/start_tracker/tasks/main.yml b/roles/admin/start_tracker/tasks/main.yml index 14a1b28..29186f8 100644 --- a/roles/admin/start_tracker/tasks/main.yml +++ b/roles/admin/start_tracker/tasks/main.yml @@ -1,5 +1,5 @@ -# Starts the Bittorrent tracker (essentially a mini-web-server) listening on port 8998. +# Starts the Bittorrent tracker (essentially a mini-web-server) - name: start tracker - shell: SCREENRC=/dev/null SYSSCREENRC=/dev/null screen -dmS murder_tracker python {{ remote_murder_path }}/murder_tracker.py && sleep 0.2 + shell: SCREENRC=/dev/null SYSSCREENRC=/dev/null screen -dmS bittorrent-tracker {{ remote_bittorrent_tracker_path }}/node_modules/bittorrent-tracker/bin/cmd.js --http -p 6969 && sleep 0.2 tags: start_tracker diff --git a/roles/admin/stop_all_peering/tasks/main.yml b/roles/admin/stop_all_peering/tasks/main.yml index db7625a..eb9f981 100644 --- a/roles/admin/stop_all_peering/tasks/main.yml +++ b/roles/admin/stop_all_peering/tasks/main.yml @@ -1,7 +1,7 @@ # Sometimes peers can go on forever (usually because of an error). # This command will forcibly kill all "murder_client.py peer" commands that are running. - name: stop all peering - shell: pkill -f "murder_client.py peer" + shell: pkill -f "aria2c" register: result failed_when: result.rc not in [0, 1] changed_when: result.rc not in [0, 1] diff --git a/roles/admin/stop_tracker/tasks/main.yml b/roles/admin/stop_tracker/tasks/main.yml index bea475d..0187642 100644 --- a/roles/admin/stop_tracker/tasks/main.yml +++ b/roles/admin/stop_tracker/tasks/main.yml @@ -1,8 +1,9 @@ # If the Bittorrent tracker is running, this will kill the process. Note that if it is not running you will receive an error. - name: stop tracker - shell: pkill -f 'SCREEN.*murder_tracker.py' + shell: pkill -f 'SCREEN.*bittorrent-tracker*' register: result failed_when: result.rc not in [0, 1] changed_when: result.rc not in [0, 1] tags: stop_tracker + diff --git a/roles/murder/common/tasks/main.yml b/roles/murder/common/tasks/main.yml index 44c28a3..9d74a0c 100644 --- a/roles/murder/common/tasks/main.yml +++ b/roles/murder/common/tasks/main.yml @@ -1,7 +1,15 @@ - name: set tagname to filename - set_fact: filename={{ default_temp_path }}/{{ tag }}.tar.gz + set_fact: filename={{ destination_path }}/{{ tag }}.tar.gz tags: common +- name: set tagname to source filename + set_fact: source_filename={{ seeder_files_path }}/{{ tag }}.tar.gz + tags: common + + +- name: set tagname to base filename + set_fact: base_filename={{ tag }}.tar.gz + tags: common diff --git a/roles/murder/create_torrent/tasks/main.yml b/roles/murder/create_torrent/tasks/main.yml index 53a18d1..3149698 100644 --- a/roles/murder/create_torrent/tasks/main.yml +++ b/roles/murder/create_torrent/tasks/main.yml @@ -1,21 +1,47 @@ -- name: create tar.gz - shell: tar -c -C {{ seeder_files_path }}/ {{ tar_options }} .|pigz > {{ filename }} +#- name: create tar.gz +# shell: tar -c -C {{ seeder_files_path }}/ {{ tar_options }} .|pigz > {{ filename }} +# when: node_type == "seeder" +# tags: create_tgz + +- name: remount cifs share + shell: systemctl restart SL{{ seeder_ADABO }}CIFSBDIG_S.service when: node_type == "seeder" - tags: create_tgz + tags: create_torrent + become: yes - name: set tracker ip set_fact: tracker_host={{ hostvars[groups['tracker'][0]]['ansible_default_ipv4']['address'] }} when: node_type == "seeder" tags: create_torrent -- name: create torrent - shell: python {{ remote_murder_path }}/murder_make_torrent.py '{{ filename }}' {{ tracker_host }}:{{ tracker_port }} '{{ filename }}.torrent' +- name: remove torrent + shell: rm '/tmp/{{ base_filename }}.torrent' when: node_type == "seeder" tags: create_torrent -- name: fetch torrent - fetch: src={{ filename }}.torrent dest={{ torrent_path }}/ flat=yes fail_on_missing=yes +- name: install mktorrent + yum: name=mktorrent state=latest when: node_type == "seeder" - tags: fetch_torrent + tags: create_torrent +- name: create torrent + shell: mktorrent -l 21 -a "http://{{ tracker_host }}:{{ tracker_port }}/announce" -o '/tmp/{{ base_filename }}.torrent' '{{ source_filename }}' + when: node_type == "seeder" + tags: create_torrent + +- name: chmod torrent file + shell: chmod a+r '/tmp/{{ base_filename }}.torrent' + when: node_type == "seeder" + tags: create_torrent + +#- name: create torrent +# shell: python {{ remote_murder_path }}/murder_make_torrent.py '{{ source_filename }}' {{ tracker_host }}:{{ tracker_port }} '/tmp/{{ base_filename }}.torrent' +# when: node_type == "seeder" +# tags: create_torrent + +#- name: fetch torrent +#- name: fetch torrent +# fetch: src={{ base_filename }}.torrent dest={{ ansible_cache_path }}/ flat=yes fail_on_missing=yes +# when: node_type == "seeder" +# tags: fetch_torrent diff --git a/roles/murder/peer/tasks/main.yml b/roles/murder/peer/tasks/main.yml index 2ef37d9..843174b 100644 --- a/roles/murder/peer/tasks/main.yml +++ b/roles/murder/peer/tasks/main.yml @@ -21,6 +21,11 @@ when: unsafe_please_delete tags: peer +- name: supprime + shell: /opt/{{ peer_ADABO }}/Logiciel/BDIG/Specifique_CAT/Scripts/clone_torrent_etape_supprime.sh + when: not only_deploy_archive + tags: peer + - name: check destination_path/* shell: | ( @@ -33,14 +38,58 @@ tags: peer - name: upload torrent file - copy: src={{ torrent_path }}/{{ filename|basename }}.torrent dest={{ filename }}.torrent + copy: src=/tmp/{{ filename|basename }}.torrent dest={{ filename }}.torrent tags: peer -- name: run murder_client.py - shell: python {{ remote_murder_path }}/murder_client.py peer '{{ filename }}.torrent' '{{filename}}' {{ ansible_default_ipv4['address'] }} +- name: copy aria2 RHEL6 + copy: src={{ src_path }}/packages/aria2-1.16.4-1.el6.rf.x86_64.rpm dest="{{ ansible_cache_path }}/" + tags: peer + when: ansible_distribution_major_version == "6" + +- name: copy nettle RHEL6 + copy: src={{ src_path }}/packages/nettle-2.2-1.el6.rf.x86_64.rpm dest="{{ ansible_cache_path }}/" + tags: peer + when: ansible_distribution_major_version == "6" + +- name: copy aria2 RHEL7 + copy: src={{ src_path }}/packages/aria2-1.18.10-2.el7.1.x86_64.rpm dest="{{ ansible_cache_path }}/" + tags: peer + when: ansible_distribution_major_version == "7" + +- name: install aria2 RHEL6 + shell: yum -y localinstall "{{ ansible_cache_path }}"/aria2*.rpm "{{ ansible_cache_path }}"/nettle*.rpm --nogpgcheck + tags: peer + when: ansible_distribution_major_version == "6" + +- name: install aria2 RHEL7 + shell: yum -y localinstall "{{ ansible_cache_path }}"/aria2*.rpm --nogpgcheck + tags: peer + when: ansible_distribution_major_version == "7" + +- name: clean existing files + shell: rm -rf {{ filename }} {{ filename }}.aria2 + when: remove_existing tags: peer -- name: tar xf +- name: run aria2 + shell: aria2c --console-log-level=notice --peer-id-prefix={{ inventory_hostname[-8:] }} --disable-ipv6 --enable-dht=false --listen-port=6881 --allow-overwrite=true --bt-max-peers=4 --enable-peer-exchange=true --bt-seed-unverified=true --seed-time=1 -d '{{ destination_path }}' '{{ filename }}.torrent' &> /var/log/{{ peer_ADABO }}/BDIG/torrent.log +# shell: aria2c --console-log-level=debug --disable-ipv6 --enable-dht=false --listen-port=6881 --allow-overwrite=true --bt-max-peers=4 --enable-peer-exchange=true --bt-seed-unverified=true --seed-time=1 -d '{{ destination_path }}' '{{ filename }}.torrent' &> /var/log/{{ peer_ADABO }}/BDIG/torrent.log + tags: peer + +#- name: run murder_client.py +# shell: python {{ remote_murder_path }}/murder_client.py peer '{{ filename }}.torrent' '{{filename}}' {{ ansible_default_ipv4['address'] }} +# tags: peer + +- name: tar xf shell: unpigz -c {{ filename }} | tar -x -C "{{ destination_path }}" + when: not only_deploy_archive tags: peer +- name: clean downloaded files + shell: rm -rf {{ filename }} {{ filename }}.torrent + tags: peer + +- name: clone_torrent.sh + shell: /opt/{{ peer_ADABO }}/Logiciel/BDIG/Specifique_CAT/Scripts/clone_torrent.sh + when: not only_deploy_archive + tags: peer diff --git a/roles/murder/rm_tgz/meta/main.yml b/roles/murder/rm_tgz/meta/main.yml deleted file mode 100644 index 280b002..0000000 --- a/roles/murder/rm_tgz/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - { role: murder/common } diff --git a/roles/murder/rm_tgz/tasks/main.yml b/roles/murder/rm_tgz/tasks/main.yml deleted file mode 100644 index bf822bc..0000000 --- a/roles/murder/rm_tgz/tasks/main.yml +++ /dev/null @@ -1,7 +0,0 @@ - - - -- name: run {{ filename }} - file: path='{{ filename }}' state=absent - tags: rm_tgz - diff --git a/roles/murder/start_seeding/tasks/main.yml b/roles/murder/start_seeding/tasks/main.yml index d7bec1f..a903752 100644 --- a/roles/murder/start_seeding/tasks/main.yml +++ b/roles/murder/start_seeding/tasks/main.yml @@ -3,6 +3,10 @@ # The ip address returned by the 'host' bash command will be announced to the tracker. # The server will not stop seeding until the stop_seeding task is called. # You must specify a valid 'tag' argument (which identifies the .torrent in /tmp to use) +#- name: start seeding +# shell: SCREENRC=/dev/null SYSSCREENRC=/dev/null screen -dmS 'seeder-{{ tag }}' python {{ remote_murder_path }}/murder_client.py seeder '/tmp/{{ base_filename }}.torrent' '{{ seeder_files_path }}/{{ base_filename }}' {{ ansible_default_ipv4['address'] }} +# tags: start_seeding + - name: start seeding - shell: SCREENRC=/dev/null SYSSCREENRC=/dev/null screen -dmS 'seeder-{{ tag }}' python {{ remote_murder_path }}/murder_client.py seeder '{{ filename }}.torrent' '{{ filename }}' {{ ansible_default_ipv4['address'] }} + shell: SCREENRC=/dev/null SYSSCREENRC=/dev/null screen -dmS 'seeder-{{ tag }}' aria2c --console-log-level=notice --peer-id-prefix={{ inventory_hostname[-8:] }} --disable-ipv6 --enable-dht=false --listen-port=6881 --allow-overwrite=true --bt-max-peers=2 --bt-seed-unverified=true --seed-ratio=0.0 -d '{{ seeder_files_path }}' '/tmp/{{ base_filename }}.torrent' tags: start_seeding diff --git a/setup.yml b/setup.yml index eda2676..edab417 100644 --- a/setup.yml +++ b/setup.yml @@ -1,20 +1,29 @@ - -- name: get distribute_files - hosts: ansible_host - roles: - - admin/clone_murder_files - -- name: Upload the support files and install pigz +- name: Prepare ansible cache + hosts: + - tracker + - seeder + - peer + sudo: yes + roles: + - admin/prepare + +- name: Install bittorrent-tracker + hosts: + - tracker + sudo: yes + roles: + - admin/deploy_bittorrent-tracker + +- name: Install pigz hosts: - tracker - seeder - peer sudo: yes roles: - - admin/distribute_files - admin/pigz -- name: install screen +- name: Install screen hosts: - tracker - seeder diff --git a/vcsim_vars-example.yml b/vcsim_vars-example.yml new file mode 100644 index 0000000..55eb375 --- /dev/null +++ b/vcsim_vars-example.yml @@ -0,0 +1,9 @@ +vcenter_servers: + - SWTTSV1VCSSA.sres.stech + - SWTTYV1VCSYA.yres.ytech + - SWMUZV1VCSZA.zres.ztech + - SWMUZV1VCSZB.zres.ztech + - SWMUZV1VCSZC.zres.ztech + - SWMUZV1VCSZD.zres.ztech +vcenter_user: user +vcenter_pass: "password" diff --git a/vm_list.yml b/vm_list.yml new file mode 100644 index 0000000..df236ea --- /dev/null +++ b/vm_list.yml @@ -0,0 +1,6 @@ +vm_list: +# - HLZURT1EXE02 + - HLZURT1EXE04 + - HLZURT1EXE05 + - HLZURT1EXE06 + - HLZURT1EXE08