Moving deployment and configuration scripts over to ansible, and general updates

Author: andrewebdev

README.textile

@@ -0,0 +1,3 @@
+
+This is just the skeleton that I use for easilly starting a new project.
+

buildout.cfg

@@ -21,22 +21,16 @@ eggs =
 # The versions below are as example only, please change these to use
 # latest versions of packages you require
 [versions]
-pillow = 1.7.5
-django = 1.4.3
-gunicorn = 0.16.1
+pillow = 2.6.1
+django = 1.7.1
+gunicorn = 19.1.1
+psycopg2 = 2.5.4
 
 
 [sources]
 # Sources from repos here. See mr.developer documentation
 
 
-[logs]
-access_log = ${buildout:directory}/log/access.log
-instance_log = ${buildout:directory}/log/instance.log
-error_log = ${buildout:directory}/log/error.log
-supervisor_log = ${buildout:directory/log/supervisor.log
-
-
 [python]
 recipe = zc.recipe.egg
 interpreter = python

fabfile/__init__.py

@@ -0,0 +1,17 @@
+[local]
+localhost ansible_connection=local
+
+[webservers]
+# Webservers here
+
+[webservers:vars]
+git_branch = production
+server_name = www.example.com
+listen_port = 
+user_home = /home/{{ ansible_ssh_user }}
+webapp_name = <your_django_app_name>
+webapp_user = <app_username>
+webapp_dir = {{ user_home }}/webapps/{{ webapp_name }}
+webapp_socketfile = {{ webapp_dir }}/bin/{{ webapp_name }}.sock
+webapp_logs = {{ webapp_dir }}/log
+webapp_media = {{ webapp_dir }}/cdn

fabfile/production.py

@@ -0,0 +1,8 @@
+# A clean webfaction server wont have pip installed. This role will take care
+# of this. currently I'm using python 2.7, but I may in future use 3.x
+---
+- name: Ensure the local python packages dir exists
+  file: path=~/lib/python2.7 state=directory
+
+- name: Install pip
+  easy_install: executable=easy_install-2.7 name=pip

playbooks/production

@@ -0,0 +1,40 @@
+# Set up supervisor on the webfaction server and prepares the default
+# configuration.
+---
+- name: Check that required supervisor directories exist
+  file: path={{ item }} state=directory
+  with_items:
+  - ~/local/etc/supervisord
+  - ~/log/supervisord
+
+# Supervisor expects config to be located in ~/etc,
+# but we will keep it in ~/local/etc, for this reason we will symlink to the
+# correct place
+- name: Create symlink to correct config location
+  file: path=~/etc
+        src=~/local/etc
+        state=link
+
+- name: Install supervisor
+  pip: name=supervisor state=present
+       executable={{ user_home }}/bin/pip2.7
+       extra_args="--user"
+
+- name: Create shell script that checks that supervisord is running
+  template: src=check_supervisord.sh.j2
+            dest={{ user_home }}/bin/check_supervisord.sh
+            mode=0744
+
+- name: Create base superivord config
+  template: src=supervisord.conf.j2
+            dest=~/local/etc/supervisord.conf
+            mode=0644
+
+# Set up supervisor cron jobs
+- name: Cron to check if supervisor is running at reboot
+  cron: name="start supervisor at boot" special_time=reboot
+        job="/home/{{ username }}/bin/check_supervisor.sh"
+
+- name: Cron to check that supervisor is running every minute
+  cron: name="is supervisor running"
+        job="/home/{{ username }}/bin/check_supervisor.sh"

playbooks/roles/python/tasks/main.yml

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if ps -u {{ ansible_ssh_user }} | grep -E 'supervisord$' > /dev/null; then
+    echo "supervisord is running."
+else
+    echo "starting supervisord."
+    {{ user_home }}/bin/supervisord
+    {{ user_home }}/bin/supervisorctl start all
+fi

playbooks/roles/supervisor/tasks/main.yml

@@ -0,0 +1,18 @@
+[unix_http_server]
+file={{ user_home }}/tmp/supervisor.sock
+
+[supervisord]
+logfile={{ user_home }}/log/supervisord/supervisord.log
+logfile_maxbytes=50MB
+logfile_backups=10
+loglevel=info
+pidfile={{ user_home }}/tmp/supervisord.pid
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix://{{ user_home }}/tmp/supervisor.sock
+
+[include]
+files = {{ user_home }}/local/etc/supervisord/*.ini

playbooks/roles/supervisor/templates/check_supervisord.sh.j2

@@ -0,0 +1,18 @@
+---
+- name: Reload supervisord
+  supervisorctl: name={{ item }}
+                 state=present
+                 supervisorctl_path={{ user_dir }}/bin/supervisorctl
+  with_items:
+    - "{{ app_name }}"
+    - nginx
+
+- name: Restart webapp
+  supervisorctl: name={{ app_name }} state=restarted
+                 supervisorctl_path={{ user_dir }}/bin/supervisorctl
+  tags: restart_webapp
+
+- name: Restart nginx
+  supervisorctl: name=nginx state=restarted
+                 supervisorctl_path={{ user_dir }}/bin/supervisorctl
+  tags: restart_nginx

playbooks/roles/supervisor/templates/supervisord.conf.j2

@@ -0,0 +1,14 @@
+# Our webfaction tasks are mainly used to set up some final links between
+# supervisor, nginx, and our future webapp.
+# As a side note; I'm not convinced these tasks belong here, and will
+# probably move them sometime in the future.
+---
+- name: Make sure log directory exists
+  file: path=~/log state="directory"
+
+- name: Create the nginx supervisor config
+  template: src=nginx_supervisor.ini.j2
+            dest=~/local/etc/supervisord/nginx.ini
+  notify:
+  - Reload supervisord
+  - Restart nginx

playbooks/roles/webfaction/handlers/main.yml

@@ -0,0 +1,9 @@
+[program:nginx]
+command = {{ user_home }}/local/sbin/nginx
+user = {{ username }}
+stdout_logfile = {{ user_home }}/log/nginx_out.log
+stderr_logfile = {{ user_home }}/log/nginx_err.log
+autostart = true
+autorestart = true
+redirect_stderr = true
+exitcodes = 0

playbooks/roles/webfaction/tasks/main.yml

@@ -0,0 +1,77 @@
+---
+# I prefer that my hosts are unaware of the code repo, and for that reason
+# I don't pull code from a remote git repo, but rather archive it, and
+# extract the archive on the remote.
+- name: Create latest code archive
+  local_action: shell cd "../" &&
+    git archive --format=tar {{ git_branch }} | gzip >{{ webapp_name }}-{{ git_branch }}.tar.gz
+
+
+- name: Extract archive on remote
+  unarchive: src=../{{ webapp_name }}-{{ git_branch }}.tar.gz dest={{ webapp_dir }}/
+
+
+# We may not always want to run bootstrap, but the method below with never
+# run again after it was first bootstrapped. We may do something about this
+# in future. Maybe a variable or flag to force bootstrap to run?
+- name: Bootstrap webapp
+  command: python2.7 -S bootstrap.py
+  args:
+    chdir: "{{ webapp_dir }}"
+    creates: "{{ webapp_dir }}/bin/buildout"
+
+
+- name: Run buildout and install dependencies
+  command: bin/buildout
+  args:
+    chdir: "{{ webapp_dir }}"
+  tags: run_buildout
+
+
+- name: Collect django app static media
+  command: bin/{{ webapp_name }} collectstatic --noinput
+  args:
+    chdir: "{{ webapp_dir }}"
+
+
+- name: Create database tables
+  command: bin/{{ webapp_name }} syncdb --noinput
+  args:
+    chdir: "{{ webapp_dir }}"
+
+
+- name: Migrate database
+  command: bin/{{ webapp_name }} migrate
+  args:
+    chdir: "{{ webapp_dir }}"
+  notify:
+  - Restart webapp
+
+
+- name: Create webapp nginx config
+  template: src=nginx.conf.j2
+            dest=~/local/nginx/sites-enabled/{{ webapp_name }}.conf
+            mode=0644
+  notify:
+  - Restart nginx 
+
+
+- name: Create supervisor config for our webapp
+  template: src=webapp_supervisor.ini.j2
+            dest=~/local/etc/supervisord/{{ webapp_name }}.ini
+            mode=0644
+  notify:
+  - Reload supervisord
+  - Restart webapp
+
+
+## Set up our cron jobs
+# The examples here is because I normally use django mailer as a simple
+# mail Queue
+# - name: <your_app_name> - Send mail cron job
+#   cron: name="send mail"
+#         job="{{ webapp_dir }}/bin/{{ webapp_name }} send_mail >> {{ webapp_logs }}/cron_mail.log 2>&1"
+
+# - name: <your_app_name> - Retry mail cron job
+#   cron: name="retry deferred mail" minute="0,20,40" 
+#         job="{{ webapp_dir }}/bin/{{ webapp_name }} retry_deferred >> {{ webapp_logs }}/cron_mail_deferred.log 2>&1"

playbooks/roles/webfaction/templates/nginx_supervisor.ini.j2

@@ -0,0 +1,38 @@
+
+server {
+    server_name {{ server_name }};
+    listen {{ listen_port }};
+    access_log  {{ webapp_logs }}/access.log;
+    error_log {{ webapp_logs }}/error.log;
+
+    location ^~ /media/ {
+        root {{ webapp_media }};
+        expires 31d;
+    }
+
+    location ^~ /static/ {
+        root {{ webapp_media }};
+        expires 31d;
+    }
+
+    # Enable if you want to use Gunicorn
+    location / {
+        proxy_pass http://unix:{{ webapp_socketfile }}:;
+        proxy_pass_header Server;
+        proxy_set_header Host $http_host;
+        proxy_redirect off;
+
+        proxy_connect_timeout 600;
+        proxy_send_timeout 600;
+        proxy_read_timeout 600;
+        send_timeout 600;
+
+        proxy_set_header X-Scheme $scheme;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Protocol ssl;
+        set_real_ip_from 127.0.0.1;
+        real_ip_header X-Forwarded-For;
+    }
+}

playbooks/roles/website/tasks/main.yml

@@ -0,0 +1,11 @@
+[program:{{ webapp_name }}]
+command = {{ webapp_dir }}/bin/{{ webapp_name }} run_gunicorn
+    -u {{ webapp_user }}
+    -w 4
+    -b unix:{{ webapp_socketfile }}
+    --log-file={{ webapp_logs }}/{{ webapp_name }}.instance.log
+user = {{ webapp_user }}
+stdout_logfile = {{ webapp_logs }}/supervisor.log
+autostart = true
+autorestart = true
+redirect_stderr = true

playbooks/roles/website/templates/nginx.conf.j2

@@ -0,0 +1,10 @@
+---
+# This playbook only sets up a new webfaction server with specific dependencies
+# which is used in other playbooks, and any other requirements for my django
+# apps to run.
+- hosts: webservers
+  gather_facts: False
+  roles:
+    - python
+    - supervisor
+    - nginx

playbooks/roles/website/templates/webapp_supervisor.ini.j2

@@ -0,0 +1,7 @@
+# This playbook will manage the website on the webfaction
+---
+- hosts: webservers
+  gather_facts: False
+  roles:
+    - webfaction
+    - website

playbooks/server.yml

@@ -0,0 +1,17 @@
+[local]
+localhost ansible_connection=local
+
+[webservers]
+# Webservers here
+
+[webservers:vars]
+git_branch = staging
+server_name = www.example.com
+listen_port = 
+user_home = /home/{{ ansible_ssh_user }}
+webapp_name = <your_django_app_name>
+webapp_user = <app_username>
+webapp_dir = {{ user_home }}/webapps/{{ webapp_name }}
+webapp_socketfile = {{ webapp_dir }}/bin/{{ webapp_name }}.sock
+webapp_logs = {{ webapp_dir }}/log
+webapp_media = {{ webapp_dir }}/cdn