Heavily rework Ansible playbooks to use deb install supervisor.

Also changes to locations of logs, configurations and virtualenvs.

config/supervisor/mailsink_supervisor.conf

@@ -0,0 +1,5 @@
+; Mailsink configuration
+[program:mailsink]
+command=/srv/www/virtualenvs/mail_server/bin/mailsinkd -w 8025 -s 25
+stopsignal=TERM
+stderr_logfile_backups=5

config/supervisor/rookeries_supervisor.conf

@@ -0,0 +1,6 @@
+; UWSGI for Rookeries
+[program:rookeries_uwsgi]
+user=www-data
+command=/srv/www/virtualenvs/uwsgi/bin/uwsgi --ini /srv/www/config/uwsgi/rookeries-uwsgi.ini
+stopsignal=TERM
+stderr_logfile_backups = 10

config/supervisor/supervisor.conf

@@ -1,58 +0,0 @@
-; Rookeries supervisor config file.
-;
-; For more information on the config file, please see:
-; http://supervisord.org/configuration.html
-;
-; Notes:
-;  - Shell expansion ("~" or "$HOME") is not supported.  Environment
-;    variables can be expanded using this syntax: "%(ENV_HOME)s".
-;  - Comments must have a leading space: "a=b ;comment" not "a=b;comment".
-
-[unix_http_server]
-file=/tmp/supervisor.sock   ; (the path to the socket file)
-;chmod=0700                 ; socket file mode (default 0700)
-;chown=nobody:nogroup       ; socket file uid:gid owner
-;username=user              ; (default is no username (open server))
-;password=123               ; (default is no password (open server))
-
-;[inet_http_server]         ; inet (TCP) server disabled by default
-;port=127.0.0.1:9001        ; (ip_address:port specifier, *:port for all iface)
-;username=user              ; (default is no username (open server))
-;password=123               ; (default is no password (open server))
-
-[supervisord]
-logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)
-logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
-logfile_backups=10           ; (num of main logfile rotation backups;default 10)
-loglevel=info                ; (log level;default info; others: debug,warn,trace)
-pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-nodaemon=false               ; (start in foreground if true;default false)
-minfds=1024                  ; (min. avail startup file descriptors;default 1024)
-minprocs=200                 ; (min. avail process descriptors;default 200)
-;umask=022                   ; (process file creation umask;default 022)
-;user=chrism                 ; (default is current user, required if root)
-;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')
-;directory=/tmp              ; (default is not to cd during start)
-;nocleanup=true              ; (don't clean up tempfiles at start;default false)
-;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)
-;environment=KEY="value"     ; (key value pairs to add to environment)
-;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
-
-; Mailsink configuration
-[program:mailsink]
-command=/srv/www/mail_server/bin/mailsinkd -w 8025 -s 25
-stopsignal=TERM
-
-; UWSGI for Rookeries
-[program:rookeries_uwsgi]
-command=/srv/www/uwsgi/bin/uwsgi --ini /srv/www/config/uwsgi/rookeries-uwsgi.ini
-stopsignal=TERM

config/uwsgi/rookeries-uwsgi.ini

@@ -1,6 +1,8 @@
 [uwsgi]
 socket = :8001
-log = /tmp/rookeries-uwsgi.log
 master = true
+processes = 8
+
+chdir = /srv/www/rookeries_webapp
+virtualenv = /srv/www/virtualenvs/rookeries
 module = rookeries:make_rookeries_app()
-virtualenv = /srv/www/rookeries

deployment.yaml

@@ -2,4 +2,3 @@
 - include: deployment/rookeries_install.yaml
 - include: deployment/uwsgi.yaml
 - include: deployment/email_server.yaml
-- include: deployment/supervisor.yaml

deployment/email_server.yaml

@@ -1,9 +1,13 @@
 ---
 - hosts: all
+  sudo: yes
   vars:
       web_server_home: /srv/www
-      email_server_venv: "{{ web_server_home }}/mail_server"
+      email_server_venv: "{{ web_server_home }}/virtualenvs/mail_server"
   tasks:
 
-    - name: PIP install mailsink test server into target virtualenv
+    - name: install mailsink test server into target virtualenv
       pip: name=mailsink virtualenv={{ email_server_venv }} version=0.0.2
+
+    - name: restart up the mailsink program
+      supervisorctl: name=mailsink state=restarted config=/etc/supervisor/supervisor.conf

deployment/rookeries_install.yaml

@@ -3,18 +3,23 @@
   sudo: yes
   vars:
     web_server_home: /srv/www
-    rookeries_venv: "{{ web_server_home }}/rookeries"
+    web_server_group: www-data
+    rookeries_venv: "{{ web_server_home }}/virtualenvs/rookeries"
     rookeries_package: rookeries-0.4.0.tar.bz2
   tasks:
 
   - name: upload + extract rookeries package to web server home
-    unarchive: src=../../../{{ rookeries_package }} dest={{ web_server_home }}
+    unarchive: src=../../../{{ rookeries_package }} dest={{ web_server_home }}/
 
-  - name: pip install rookeries dependencies into target virtualenv
+  - name: install rookeries dependencies
     pip: requirements=rookeries_webapp/requirements.txt virtualenv={{ rookeries_venv }} chdir={{ web_server_home }}
 
-  - name: PIP install extra MySQL python connector dependency
+  - name: install extra MySQL python connector dependency
     pip: name=mysql-connector-python virtualenv={{ rookeries_venv }} version=1.1.6
          extra_args='--allow-external mysql-connector-python --allow-unverified mysql-connector-python'
 
+  - name: fix permissions on extracted web app
+    file: path={{ web_server_home }}/rookeries_webapp owner={{ web_server_group }} group={{ web_server_group }}
+          state=directory recurse=yes
+
   # TODO Step 4 - Setup environment variables on target system related to config.

deployment/supervisor.yaml

@@ -1,42 +0,0 @@
----
-- hosts: all
-  sudo: yes
-  vars:
-      web_server_home: /srv/www
-      web_server_group: www-data
-      supervisor_config: "{{ web_server_home }}/config/supervisor"
-      supervisor_venv: "{{ web_server_home }}/supervisor"
-  tasks:
-
-     - name: PIP install supervisor into target virtualenv
-       pip: name=supervisor virtualenv={{ supervisor_venv }} version=3.1.3 extra_args=--pre
-
-     - name: Setup webapp deployment configuration folder with the correct permissions
-       file: path={{ supervisor_config }} state=directory
-             owner={{ ansible_env.SUDO_USER }} group={{ web_server_group }} mode=0774
-
-     - name: Upload rookeries supervisor configuration to web server home
-       copy: src=../config/supervisor/supervisor.conf dest={{ supervisor_config }}
-
-     - name: Link supervisor binary into /usr/local/bin
-       file: src={{ supervisor_venv }}/bin/{{ item }} dest=/usr/local/bin/{{ item }} state=link
-       with_items:
-          - supervisord
-          - supervisorctl
-
-     # TODO Separate out starting and stopping of supervisord + supervisorctl outside of provisioning
-     - name: Check if supervisord is running
-       shell: pgrep supervisor
-       ignore_errors: on
-       changed_when: off
-       register: supervisor_running
-
-     - name: Running supervisord
-       shell: "{{ supervisor_venv }}/bin/supervisord -c {{ supervisor_config }}/supervisor.conf"
-       when: supervisor_running|failed
-
-     - name: Start up the mailsink program
-       supervisorctl: name=mailsink state=restarted config={{ supervisor_config }}/supervisor.conf
-
-     - name: Start up the uwsgi program
-       supervisorctl: name=uwsgi state=restarted config={{ supervisor_config }}/supervisor.conf

deployment/uwsgi.yaml

@@ -1,20 +1,18 @@
 ---
 - hosts: all
-  sudo: yes
   vars:
       web_server_home: /srv/www
       web_server_group: www-data
-      uwsgi_venv: "{{ web_server_home }}/uwsgi"
+      uwsgi_venv: "{{ web_server_home }}/virtualenvs/uwsgi"
       uwsgi_config: "{{ web_server_home }}/config/uwsgi"
       rookeries_uwsgi_ini: rookeries-uwsgi.ini
   tasks:
 
-     - name: PIP install UWSGI server into target virtualenv
+     - name: install uwsgi server
        pip: name=uwsgi virtualenv={{ uwsgi_venv }} version=2.0
 
-     - name: Setup webapp deployment configuration folder with the correct permissions
+     - name: upload rookeries configuration
-       file: path={{ uwsgi_config }} state=directory
-             owner={{ ansible_env.SUDO_USER }} group={{ web_server_group }} mode=0774
-
-     - name: Upload rookeries UWSGI configuration to web server home
        copy: src=../config/uwsgi/{{ rookeries_uwsgi_ini }} dest={{ uwsgi_config }}
+
+     - name: restart uwsgi server
+       supervisorctl: name=uwsgi state=restarted config=/etc/supervisor/supervisor.conf

pre_deployment.yaml

@@ -2,6 +2,9 @@
 - include: pre_deployment/base_linux.yaml
 - include: pre_deployment/python.yaml
 - include: pre_deployment/mysql_db.yaml
+- include: pre_deployment/nginx.yaml
+- include: pre_deployment/supervisor.yaml
 - include: pre_deployment/web_data_folders.yaml
 - include: pre_deployment/nodejs.yaml
-- include: pre_deployment/nginx.yaml
+
+# TODO Add in globally defined values for webapp and virtualenv folders

pre_deployment/base_linux.yaml

@@ -3,8 +3,8 @@
   sudo: yes
   tasks:
 
-  - name: Install vim
+  - name: install vim
     apt: pkg=vim state=present
 
-  - name: Configure vim
+  - name: configure vim
     copy: src=../config/base/_vimrc dest=/home/vagrant/.vimrc

pre_deployment/mysql_db.yaml

@@ -8,16 +8,16 @@
         password: system_admin
   tasks:
 
-  - name: Setup MySQL 5 server + Python dependencies
+  - name: setup MySQL server + Python dependencies
     apt: pkg={{ item }} state=present
     with_items:
       - mysql-server
       - python-mysqldb
 
-  - name: Setup MySQL databases for rookeries.
+  - name: setup MySQL databases for rookeries.
     mysql_db: name={{ item.key }} state=present
     with_dict: databases
 
-  - name: Setup MySQL database users for rookeries.
+  - name: add MySQL database users for rookeries.
     mysql_user: name={{ item.value.username }} password={{ item.value.password }} priv={{ item.key }}:ALL
     with_dict: databases

pre_deployment/nginx.yaml

@@ -6,23 +6,30 @@
     web_server_group: www-data
   tasks:
 
-  - name: Setup nginx webserver
+  - name: install nginx webserver
     apt: pkg=nginx-full state=present
 
-  - name: Configure nginx
+  - name: configure nginx
     copy: src=../config/nginx/{{ rookeries_nginx_conf }} dest=/etc/nginx/sites-available
 
-  - name: Link the rookeries uwsgi file
+  - name: enable the rookeries nginx configuration
     file: state=link
           src=/etc/nginx/sites-available/{{ rookeries_nginx_conf }}
           path=/etc/nginx/sites-enabled/{{ rookeries_nginx_conf }}
 
-  - name: Unlink the default page
+  - name: disable the default page configuration
     file: state=absent path=/etc/nginx/sites-enabled/default
 
-  - name: Run nginx service
+  - name: restart nginx
     service: name=nginx state=restarted
 
-  - name: Change permissions on nginx logs
+  - name: set permissions on nginx log directory
     file: path=/var/log/nginx state=directory
-          owner={{ web_server_group }} group={{ web_server_group }} mode=0766
+          owner={{ web_server_group }} group={{ web_server_group }} mode=0755
+
+  - name: set permissions on nginx logs
+    file: path=/var/log/nginx/{{ item }} state=file
+          owner={{ web_server_group }} group={{ web_server_group }} mode=0644
+    with_items:
+        - access.log
+        - error.log

pre_deployment/nodejs.yaml

@@ -3,7 +3,7 @@
   sudo: yes
   tasks:
 
-  - name: Setup nodejs + npm dependencies
+  - name: install nodejs + npm dependencies
     apt: pkg={{ item }} state=present
     with_items:
         - nodejs
@@ -12,7 +12,7 @@
   - name: link nodejs binary correctly
     file: src=/usr/bin/nodejs dest=/usr/bin/node state=link
 
-  - name: Setup coffeescript + lessc dependency
+  - name: install coffeescript + less dependencies
     npm: name={{ item }} global=yes state=present
     with_items:
         - coffee-script

pre_deployment/python.yaml

@@ -3,16 +3,14 @@
   sudo: yes
   tasks:
 
-  - name: Setup Python setuptools dependencies
+  - name: install Python setuptools dependencies
     apt: pkg={{ item }} state=present
     with_items:
       - python-dev
       - python-setuptools
 
-  - name: Bootstrap PIP using Setuptools
+  - name: bootstrap PIP using setuptools
     easy_install: name=pip
 
-  - name: Basic Python dependencies
+  - name: install Python virtualenv
-    pip: name={{ item }}
+    pip: name=virtualenv
-    with_items:
-      - virtualenv

pre_deployment/supervisor.yaml

@@ -0,0 +1,30 @@
+---
+- hosts: all
+  sudo: yes
+  vars:
+      web_server_group: www-data
+      supervisor_configs:
+        - mailsink_supervisor.conf
+        - rookeries_supervisor.conf
+  tasks:
+
+     - name: install supervisord
+       apt: name=supervisor state=present
+
+     - name: set permissions on supervisor config and log directories
+       file: path={{ item }} group={{ web_server_group }} state=directory mode=0755
+       with_items:
+          - /etc/supervisor
+          - /etc/supervisor/conf.d
+          - /var/log/supervisor
+
+     - name: modify supervisor configuration
+       ini_file: dest=/etc/supervisor/supervisord.conf section=unix_http_server
+                 option={{ item.key }} value={{ item.value }}
+       with_dict:
+           chmod: "0770"
+           chown: root:www-data
+
+     - name: upload rookeries supervisor configuration to web server home
+       copy: src=../config/supervisor/{{ item }} dest=/etc/supervisor/conf.d
+       with_items: supervisor_configs

pre_deployment/web_data_folders.yaml

@@ -15,8 +15,20 @@
           owner={{ web_server_group }} group={{ web_server_group }} mode=0774
     with_items:
         - config
+        - config/uwsgi
         - logs
-        - logs/nginx
+        - virtualenvs
+
+  - name: setup webapp deployment configuration + log folders with the correct permissions
+    file: state=link
+          path={{ web_app_home }}/{{ item.value }}
+          src={{ item.key }}
+          owner={{ web_server_group }} group={{ web_server_group }} mode=0774
+    with_dict:
+        /etc/nginx: config/nginx
+        /etc/supervisor: config/supervisor
+        /var/log/nginx: logs/nginx
+        /var/log/supervisor: logs/supervisor
 
   - name: add user to www-data
     user: name={{ user }} append=yes groups={{ web_server_group }}