From d5e32c5ab80b642467f952b0090c781f4a749da3 Mon Sep 17 00:00:00 2001 From: Dorian Pula Date: Wed, 3 Dec 2014 22:09:38 -0500 Subject: [PATCH] Heavily rework Ansible playbooks to use deb install supervisor. Also changes to locations of logs, configurations and virtualenvs. --- config/supervisor/mailsink_supervisor.conf | 5 ++ config/supervisor/rookeries_supervisor.conf | 6 +++ config/supervisor/supervisor.conf | 58 --------------------- config/uwsgi/rookeries-uwsgi.ini | 6 ++- deployment.yaml | 1 - deployment/email_server.yaml | 8 ++- deployment/rookeries_install.yaml | 13 +++-- deployment/supervisor.yaml | 42 --------------- deployment/uwsgi.yaml | 14 +++-- pre_deployment.yaml | 5 +- pre_deployment/base_linux.yaml | 4 +- pre_deployment/mysql_db.yaml | 6 +-- pre_deployment/nginx.yaml | 21 +++++--- pre_deployment/nodejs.yaml | 4 +- pre_deployment/python.yaml | 10 ++-- pre_deployment/supervisor.yaml | 30 +++++++++++ pre_deployment/web_data_folders.yaml | 14 ++++- 17 files changed, 108 insertions(+), 139 deletions(-) create mode 100644 config/supervisor/mailsink_supervisor.conf create mode 100644 config/supervisor/rookeries_supervisor.conf delete mode 100644 config/supervisor/supervisor.conf delete mode 100644 deployment/supervisor.yaml create mode 100644 pre_deployment/supervisor.yaml diff --git a/config/supervisor/mailsink_supervisor.conf b/config/supervisor/mailsink_supervisor.conf new file mode 100644 index 0000000..8244f29 --- /dev/null +++ b/config/supervisor/mailsink_supervisor.conf @@ -0,0 +1,5 @@ +; Mailsink configuration +[program:mailsink] +command=/srv/www/virtualenvs/mail_server/bin/mailsinkd -w 8025 -s 25 +stopsignal=TERM +stderr_logfile_backups=5 diff --git a/config/supervisor/rookeries_supervisor.conf b/config/supervisor/rookeries_supervisor.conf new file mode 100644 index 0000000..571ac7f --- /dev/null +++ b/config/supervisor/rookeries_supervisor.conf @@ -0,0 +1,6 @@ +; UWSGI for Rookeries +[program:rookeries_uwsgi] +user=www-data +command=/srv/www/virtualenvs/uwsgi/bin/uwsgi --ini /srv/www/config/uwsgi/rookeries-uwsgi.ini +stopsignal=TERM +stderr_logfile_backups = 10 diff --git a/config/supervisor/supervisor.conf b/config/supervisor/supervisor.conf deleted file mode 100644 index af13ab9..0000000 --- a/config/supervisor/supervisor.conf +++ /dev/null @@ -1,58 +0,0 @@ -; Rookeries supervisor config file. -; -; For more information on the config file, please see: -; http://supervisord.org/configuration.html -; -; Notes: -; - Shell expansion ("~" or "$HOME") is not supported. Environment -; variables can be expanded using this syntax: "%(ENV_HOME)s". -; - Comments must have a leading space: "a=b ;comment" not "a=b;comment". - -[unix_http_server] -file=/tmp/supervisor.sock ; (the path to the socket file) -;chmod=0700 ; socket file mode (default 0700) -;chown=nobody:nogroup ; socket file uid:gid owner -;username=user ; (default is no username (open server)) -;password=123 ; (default is no password (open server)) - -;[inet_http_server] ; inet (TCP) server disabled by default -;port=127.0.0.1:9001 ; (ip_address:port specifier, *:port for all iface) -;username=user ; (default is no username (open server)) -;password=123 ; (default is no password (open server)) - -[supervisord] -logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log) -logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) -logfile_backups=10 ; (num of main logfile rotation backups;default 10) -loglevel=info ; (log level;default info; others: debug,warn,trace) -pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -nodaemon=false ; (start in foreground if true;default false) -minfds=1024 ; (min. avail startup file descriptors;default 1024) -minprocs=200 ; (min. avail process descriptors;default 200) -;umask=022 ; (process file creation umask;default 022) -;user=chrism ; (default is current user, required if root) -;identifier=supervisor ; (supervisord identifier, default is 'supervisor') -;directory=/tmp ; (default is not to cd during start) -;nocleanup=true ; (don't clean up tempfiles at start;default false) -;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) -;environment=KEY="value" ; (key value pairs to add to environment) -;strip_ansi=false ; (strip ansi escape codes in logs; def. false) - -; the below section must remain in the config file for RPC -; (supervisorctl/web interface) to work, additional interfaces may be -; added by defining them in separate rpcinterface: sections -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket - -; Mailsink configuration -[program:mailsink] -command=/srv/www/mail_server/bin/mailsinkd -w 8025 -s 25 -stopsignal=TERM - -; UWSGI for Rookeries -[program:rookeries_uwsgi] -command=/srv/www/uwsgi/bin/uwsgi --ini /srv/www/config/uwsgi/rookeries-uwsgi.ini -stopsignal=TERM diff --git a/config/uwsgi/rookeries-uwsgi.ini b/config/uwsgi/rookeries-uwsgi.ini index 1debb10..aa41ae5 100644 --- a/config/uwsgi/rookeries-uwsgi.ini +++ b/config/uwsgi/rookeries-uwsgi.ini @@ -1,6 +1,8 @@ [uwsgi] socket = :8001 -log = /tmp/rookeries-uwsgi.log master = true +processes = 8 + +chdir = /srv/www/rookeries_webapp +virtualenv = /srv/www/virtualenvs/rookeries module = rookeries:make_rookeries_app() -virtualenv = /srv/www/rookeries diff --git a/deployment.yaml b/deployment.yaml index e605f7a..68b5aa0 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -2,4 +2,3 @@ - include: deployment/rookeries_install.yaml - include: deployment/uwsgi.yaml - include: deployment/email_server.yaml -- include: deployment/supervisor.yaml diff --git a/deployment/email_server.yaml b/deployment/email_server.yaml index 44c4e2e..ad618e4 100644 --- a/deployment/email_server.yaml +++ b/deployment/email_server.yaml @@ -1,9 +1,13 @@ --- - hosts: all + sudo: yes vars: web_server_home: /srv/www - email_server_venv: "{{ web_server_home }}/mail_server" + email_server_venv: "{{ web_server_home }}/virtualenvs/mail_server" tasks: - - name: PIP install mailsink test server into target virtualenv + - name: install mailsink test server into target virtualenv pip: name=mailsink virtualenv={{ email_server_venv }} version=0.0.2 + + - name: restart up the mailsink program + supervisorctl: name=mailsink state=restarted config=/etc/supervisor/supervisor.conf diff --git a/deployment/rookeries_install.yaml b/deployment/rookeries_install.yaml index bf0d4e5..c19fea9 100644 --- a/deployment/rookeries_install.yaml +++ b/deployment/rookeries_install.yaml @@ -3,18 +3,23 @@ sudo: yes vars: web_server_home: /srv/www - rookeries_venv: "{{ web_server_home }}/rookeries" + web_server_group: www-data + rookeries_venv: "{{ web_server_home }}/virtualenvs/rookeries" rookeries_package: rookeries-0.4.0.tar.bz2 tasks: - name: upload + extract rookeries package to web server home - unarchive: src=../../../{{ rookeries_package }} dest={{ web_server_home }} + unarchive: src=../../../{{ rookeries_package }} dest={{ web_server_home }}/ - - name: pip install rookeries dependencies into target virtualenv + - name: install rookeries dependencies pip: requirements=rookeries_webapp/requirements.txt virtualenv={{ rookeries_venv }} chdir={{ web_server_home }} - - name: PIP install extra MySQL python connector dependency + - name: install extra MySQL python connector dependency pip: name=mysql-connector-python virtualenv={{ rookeries_venv }} version=1.1.6 extra_args='--allow-external mysql-connector-python --allow-unverified mysql-connector-python' + - name: fix permissions on extracted web app + file: path={{ web_server_home }}/rookeries_webapp owner={{ web_server_group }} group={{ web_server_group }} + state=directory recurse=yes + # TODO Step 4 - Setup environment variables on target system related to config. diff --git a/deployment/supervisor.yaml b/deployment/supervisor.yaml deleted file mode 100644 index 9c41f31..0000000 --- a/deployment/supervisor.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- hosts: all - sudo: yes - vars: - web_server_home: /srv/www - web_server_group: www-data - supervisor_config: "{{ web_server_home }}/config/supervisor" - supervisor_venv: "{{ web_server_home }}/supervisor" - tasks: - - - name: PIP install supervisor into target virtualenv - pip: name=supervisor virtualenv={{ supervisor_venv }} version=3.1.3 extra_args=--pre - - - name: Setup webapp deployment configuration folder with the correct permissions - file: path={{ supervisor_config }} state=directory - owner={{ ansible_env.SUDO_USER }} group={{ web_server_group }} mode=0774 - - - name: Upload rookeries supervisor configuration to web server home - copy: src=../config/supervisor/supervisor.conf dest={{ supervisor_config }} - - - name: Link supervisor binary into /usr/local/bin - file: src={{ supervisor_venv }}/bin/{{ item }} dest=/usr/local/bin/{{ item }} state=link - with_items: - - supervisord - - supervisorctl - - # TODO Separate out starting and stopping of supervisord + supervisorctl outside of provisioning - - name: Check if supervisord is running - shell: pgrep supervisor - ignore_errors: on - changed_when: off - register: supervisor_running - - - name: Running supervisord - shell: "{{ supervisor_venv }}/bin/supervisord -c {{ supervisor_config }}/supervisor.conf" - when: supervisor_running|failed - - - name: Start up the mailsink program - supervisorctl: name=mailsink state=restarted config={{ supervisor_config }}/supervisor.conf - - - name: Start up the uwsgi program - supervisorctl: name=uwsgi state=restarted config={{ supervisor_config }}/supervisor.conf diff --git a/deployment/uwsgi.yaml b/deployment/uwsgi.yaml index 497862f..24770b2 100644 --- a/deployment/uwsgi.yaml +++ b/deployment/uwsgi.yaml @@ -1,20 +1,18 @@ --- - hosts: all - sudo: yes vars: web_server_home: /srv/www web_server_group: www-data - uwsgi_venv: "{{ web_server_home }}/uwsgi" + uwsgi_venv: "{{ web_server_home }}/virtualenvs/uwsgi" uwsgi_config: "{{ web_server_home }}/config/uwsgi" rookeries_uwsgi_ini: rookeries-uwsgi.ini tasks: - - name: PIP install UWSGI server into target virtualenv + - name: install uwsgi server pip: name=uwsgi virtualenv={{ uwsgi_venv }} version=2.0 - - name: Setup webapp deployment configuration folder with the correct permissions - file: path={{ uwsgi_config }} state=directory - owner={{ ansible_env.SUDO_USER }} group={{ web_server_group }} mode=0774 - - - name: Upload rookeries UWSGI configuration to web server home + - name: upload rookeries configuration copy: src=../config/uwsgi/{{ rookeries_uwsgi_ini }} dest={{ uwsgi_config }} + + - name: restart uwsgi server + supervisorctl: name=uwsgi state=restarted config=/etc/supervisor/supervisor.conf diff --git a/pre_deployment.yaml b/pre_deployment.yaml index a8bd826..60a92cc 100644 --- a/pre_deployment.yaml +++ b/pre_deployment.yaml @@ -2,6 +2,9 @@ - include: pre_deployment/base_linux.yaml - include: pre_deployment/python.yaml - include: pre_deployment/mysql_db.yaml +- include: pre_deployment/nginx.yaml +- include: pre_deployment/supervisor.yaml - include: pre_deployment/web_data_folders.yaml - include: pre_deployment/nodejs.yaml -- include: pre_deployment/nginx.yaml + +# TODO Add in globally defined values for webapp and virtualenv folders diff --git a/pre_deployment/base_linux.yaml b/pre_deployment/base_linux.yaml index 5ecb0a1..ee8dcfd 100644 --- a/pre_deployment/base_linux.yaml +++ b/pre_deployment/base_linux.yaml @@ -3,8 +3,8 @@ sudo: yes tasks: - - name: Install vim + - name: install vim apt: pkg=vim state=present - - name: Configure vim + - name: configure vim copy: src=../config/base/_vimrc dest=/home/vagrant/.vimrc diff --git a/pre_deployment/mysql_db.yaml b/pre_deployment/mysql_db.yaml index 9191fe8..e3e3f43 100644 --- a/pre_deployment/mysql_db.yaml +++ b/pre_deployment/mysql_db.yaml @@ -8,16 +8,16 @@ password: system_admin tasks: - - name: Setup MySQL 5 server + Python dependencies + - name: setup MySQL server + Python dependencies apt: pkg={{ item }} state=present with_items: - mysql-server - python-mysqldb - - name: Setup MySQL databases for rookeries. + - name: setup MySQL databases for rookeries. mysql_db: name={{ item.key }} state=present with_dict: databases - - name: Setup MySQL database users for rookeries. + - name: add MySQL database users for rookeries. mysql_user: name={{ item.value.username }} password={{ item.value.password }} priv={{ item.key }}:ALL with_dict: databases diff --git a/pre_deployment/nginx.yaml b/pre_deployment/nginx.yaml index 4b53e7e..3d89029 100644 --- a/pre_deployment/nginx.yaml +++ b/pre_deployment/nginx.yaml @@ -6,23 +6,30 @@ web_server_group: www-data tasks: - - name: Setup nginx webserver + - name: install nginx webserver apt: pkg=nginx-full state=present - - name: Configure nginx + - name: configure nginx copy: src=../config/nginx/{{ rookeries_nginx_conf }} dest=/etc/nginx/sites-available - - name: Link the rookeries uwsgi file + - name: enable the rookeries nginx configuration file: state=link src=/etc/nginx/sites-available/{{ rookeries_nginx_conf }} path=/etc/nginx/sites-enabled/{{ rookeries_nginx_conf }} - - name: Unlink the default page + - name: disable the default page configuration file: state=absent path=/etc/nginx/sites-enabled/default - - name: Run nginx service + - name: restart nginx service: name=nginx state=restarted - - name: Change permissions on nginx logs + - name: set permissions on nginx log directory file: path=/var/log/nginx state=directory - owner={{ web_server_group }} group={{ web_server_group }} mode=0766 + owner={{ web_server_group }} group={{ web_server_group }} mode=0755 + + - name: set permissions on nginx logs + file: path=/var/log/nginx/{{ item }} state=file + owner={{ web_server_group }} group={{ web_server_group }} mode=0644 + with_items: + - access.log + - error.log diff --git a/pre_deployment/nodejs.yaml b/pre_deployment/nodejs.yaml index 4d98f99..3eca965 100644 --- a/pre_deployment/nodejs.yaml +++ b/pre_deployment/nodejs.yaml @@ -3,7 +3,7 @@ sudo: yes tasks: - - name: Setup nodejs + npm dependencies + - name: install nodejs + npm dependencies apt: pkg={{ item }} state=present with_items: - nodejs @@ -12,7 +12,7 @@ - name: link nodejs binary correctly file: src=/usr/bin/nodejs dest=/usr/bin/node state=link - - name: Setup coffeescript + lessc dependency + - name: install coffeescript + less dependencies npm: name={{ item }} global=yes state=present with_items: - coffee-script diff --git a/pre_deployment/python.yaml b/pre_deployment/python.yaml index 0c14f78..8b174da 100644 --- a/pre_deployment/python.yaml +++ b/pre_deployment/python.yaml @@ -3,16 +3,14 @@ sudo: yes tasks: - - name: Setup Python setuptools dependencies + - name: install Python setuptools dependencies apt: pkg={{ item }} state=present with_items: - python-dev - python-setuptools - - name: Bootstrap PIP using Setuptools + - name: bootstrap PIP using setuptools easy_install: name=pip - - name: Basic Python dependencies - pip: name={{ item }} - with_items: - - virtualenv + - name: install Python virtualenv + pip: name=virtualenv diff --git a/pre_deployment/supervisor.yaml b/pre_deployment/supervisor.yaml new file mode 100644 index 0000000..5d1f1b4 --- /dev/null +++ b/pre_deployment/supervisor.yaml @@ -0,0 +1,30 @@ +--- +- hosts: all + sudo: yes + vars: + web_server_group: www-data + supervisor_configs: + - mailsink_supervisor.conf + - rookeries_supervisor.conf + tasks: + + - name: install supervisord + apt: name=supervisor state=present + + - name: set permissions on supervisor config and log directories + file: path={{ item }} group={{ web_server_group }} state=directory mode=0755 + with_items: + - /etc/supervisor + - /etc/supervisor/conf.d + - /var/log/supervisor + + - name: modify supervisor configuration + ini_file: dest=/etc/supervisor/supervisord.conf section=unix_http_server + option={{ item.key }} value={{ item.value }} + with_dict: + chmod: "0770" + chown: root:www-data + + - name: upload rookeries supervisor configuration to web server home + copy: src=../config/supervisor/{{ item }} dest=/etc/supervisor/conf.d + with_items: supervisor_configs diff --git a/pre_deployment/web_data_folders.yaml b/pre_deployment/web_data_folders.yaml index 893bdee..39140c6 100644 --- a/pre_deployment/web_data_folders.yaml +++ b/pre_deployment/web_data_folders.yaml @@ -15,8 +15,20 @@ owner={{ web_server_group }} group={{ web_server_group }} mode=0774 with_items: - config + - config/uwsgi - logs - - logs/nginx + - virtualenvs + + - name: setup webapp deployment configuration + log folders with the correct permissions + file: state=link + path={{ web_app_home }}/{{ item.value }} + src={{ item.key }} + owner={{ web_server_group }} group={{ web_server_group }} mode=0774 + with_dict: + /etc/nginx: config/nginx + /etc/supervisor: config/supervisor + /var/log/nginx: logs/nginx + /var/log/supervisor: logs/supervisor - name: add user to www-data user: name={{ user }} append=yes groups={{ web_server_group }}