From: Radek Czajka Date: Wed, 21 Nov 2012 15:05:20 +0000 (+0100) Subject: deployment stuff X-Git-Tag: 0.1.5~14 X-Git-Url: https://git.mdrn.pl/fnpdjango.git/commitdiff_plain/c4ec253cd69546987dc20dec4c9be9e3867986f2?ds=sidebyside;hp=9a5b05c52311da4f726fec29d8be7d173f695352 deployment stuff --- diff --git a/bin/fnpdjango_bootstrap.sh b/bin/fnpdjango_bootstrap.sh new file mode 100755 index 0000000..cb9a6d7 --- /dev/null +++ b/bin/fnpdjango_bootstrap.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +DJANGO_REQ = 'django>=1.4,<1.5' + +mkvirtualenv "$1" +pip install "$DJANGO_REQ" +django-admin.py startproject \ + --template http://pypi.nowoczesnapolska.org.pl/bootstrap/project.tar.gz \ + "$1" +pip install -r "$1"/requirements-dev.txt diff --git a/bin/git-archive-all.sh b/bin/git-archive-all.sh new file mode 100755 index 0000000..883c029 --- /dev/null +++ b/bin/git-archive-all.sh @@ -0,0 +1,209 @@ +#!/bin/bash - +# +# File: git-archive-all.sh +# +# Description: A utility script that builds an archive file(s) of all +# git repositories and submodules in the current path. +# Useful for creating a single tarfile of a git super- +# project that contains other submodules. +# +# Examples: Use git-archive-all.sh to create archive distributions +# from git repositories. To use, simply do: +# +# cd $GIT_DIR; git-archive-all.sh +# +# where $GIT_DIR is the root of your git superproject. +# +# License: GPL3 +# +############################################################################### +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +############################################################################### + +# DEBUGGING +set -e +set -C # noclobber + +# TRAP SIGNALS +trap 'cleanup' QUIT EXIT + +# For security reasons, explicitly set the internal field separator +# to newline, space, tab +OLD_IFS=$IFS +IFS=' + ' + +function cleanup () { + rm -f $TMPFILE + rm -f $TOARCHIVE + IFS="$OLD_IFS" +} + +function usage () { + echo "Usage is as follows:" + echo + echo "$PROGRAM <--version>" + echo " Prints the program version number on a line by itself and exits." + echo + echo "$PROGRAM <--usage|--help|-?>" + echo " Prints this usage output and exits." + echo + echo "$PROGRAM [--format ] [--prefix ] [--separate|-s] [output_file]" + echo " Creates an archive for the entire git superproject, and its submodules" + echo " using the passed parameters, described below." + echo + echo " If '--format' is specified, the archive is created with the named" + echo " git archiver backend. Obviously, this must be a backend that git-archive" + echo " understands. The format defaults to 'tar' if not specified." + echo + echo " If '--prefix' is specified, the archive's superproject and all submodules" + echo " are created with the prefix named. The default is to not use one." + echo + echo " If '--separate' or '-s' is specified, individual archives will be created" + echo " for each of the superproject itself and its submodules. The default is to" + echo " concatenate individual archives into one larger archive." + echo + echo " If 'output_file' is specified, the resulting archive is created as the" + echo " file named. This parameter is essentially a path that must be writeable." + echo " When combined with '--separate' ('-s') this path must refer to a directory." + echo " Without this parameter or when combined with '--separate' the resulting" + echo " archive(s) are named with a dot-separated path of the archived directory and" + echo " a file extension equal to their format (e.g., 'superdir.submodule1dir.tar')." +} + +function version () { + echo "$PROGRAM version $VERSION" +} + +# Internal variables and initializations. +readonly PROGRAM=`basename "$0"` +readonly VERSION=0.2 + +OLD_PWD="`pwd`" +TMPDIR=${TMPDIR:-/tmp} +TMPFILE=`mktemp "$TMPDIR/$PROGRAM.XXXXXX"` # Create a place to store our work's progress +TOARCHIVE=`mktemp "$TMPDIR/$PROGRAM.toarchive.XXXXXX"` +OUT_FILE=$OLD_PWD # assume "this directory" without a name change by default +SEPARATE=0 + +FORMAT=tar +PREFIX= +TREEISH=HEAD + +# RETURN VALUES/EXIT STATUS CODES +readonly E_BAD_OPTION=254 +readonly E_UNKNOWN=255 + +# Process command-line arguments. +while test $# -gt 0; do + case $1 in + --format ) + shift + FORMAT="$1" + shift + ;; + + --prefix ) + shift + PREFIX="$1" + shift + ;; + + --separate | -s ) + shift + SEPARATE=1 + ;; + + --version ) + version + exit + ;; + + -? | --usage | --help ) + usage + exit + ;; + + -* ) + echo "Unrecognized option: $1" >&2 + usage + exit $E_BAD_OPTION + ;; + + * ) + break + ;; + esac +done + +if [ ! -z "$1" ]; then + OUT_FILE="$1" + shift +fi + +# Validate parameters; error early, error often. +if [ $SEPARATE -eq 1 -a ! -d $OUT_FILE ]; then + echo "When creating multiple archives, your destination must be a directory." + echo "If it's not, you risk being surprised when your files are overwritten." + exit +elif [ `git config -l | grep -q '^core\.bare=false'; echo $?` -ne 0 ]; then + echo "$PROGRAM must be run from a git working copy (i.e., not a bare repository)." + exit +fi + +# Create the superproject's git-archive +git archive --format=$FORMAT --prefix="$PREFIX" $TREEISH > $TMPDIR/$(basename $(pwd)).$FORMAT +echo $TMPDIR/$(basename $(pwd)).$FORMAT >| $TMPFILE # clobber on purpose +superfile=`head -n 1 $TMPFILE` + +set +e +# find all '.git' dirs, these show us the remaining to-be-archived dirs +find . -name '.git' -type d -print | sed -e 's/^\.\///' -e 's/\.git$//' | grep -v '^$' >> $TOARCHIVE +set -e + +while read path; do + TREEISH=$(git submodule | grep "^ .*${path%/} " | cut -d ' ' -f 2) # git-submodule does not list trailing slashes in $path + cd "$path" + git archive --format=$FORMAT --prefix="${PREFIX}$path" ${TREEISH:-HEAD} > "$TMPDIR"/"$(echo "$path" | sed -e 's/\//./g')"$FORMAT + if [ $FORMAT == 'zip' ]; then + # delete the empty directory entry; zipped submodules won't unzip if we don't do this + zip -d "$(tail -n 1 $TMPFILE)" "${PREFIX}${path%/}" >/dev/null # remove trailing '/' + fi + echo "$TMPDIR"/"$(echo "$path" | sed -e 's/\//./g')"$FORMAT >> $TMPFILE + cd "$OLD_PWD" +done < $TOARCHIVE +# Concatenate archives into a super-archive. +if [ $SEPARATE -eq 0 ]; then + if [ $FORMAT == 'tar' ]; then + sed -e '1d' $TMPFILE | while read file; do + tar --concatenate -f "$superfile" "$file" && rm -f "$file" + done + elif [ $FORMAT == 'zip' ]; then + sed -e '1d' $TMPFILE | while read file; do + # zip incorrectly stores the full path, so cd and then grow + cd `dirname "$file"` + zip -g "$superfile" `basename "$file"` && rm -f "$file" + done + cd "$OLD_PWD" + fi + + echo "$superfile" >| $TMPFILE # clobber on purpose +fi + +while read file; do + mv "$file" "$OUT_FILE" +done < $TMPFILE diff --git a/fnpdjango/deploy/__init__.py b/fnpdjango/deploy/__init__.py new file mode 100644 index 0000000..9c7d762 --- /dev/null +++ b/fnpdjango/deploy/__init__.py @@ -0,0 +1,221 @@ +""" +Generic fabric deployment script. +Create a fabfile.py in the project and start it with: + + from fnpdjango.deploy import * + +Then set up some env properties: + project_name: slug-like project name + hosts: list of target host names + user: remote user name + app_path: where does the app go + services: list of tasks to run after deployment + +""" +from fabric.api import * +from os.path import abspath, dirname, exists, join +from fabric.contrib import files +from fabric.tasks import Task, execute + +env.virtualenv = '/usr/bin/virtualenv' +env.services = None + + +@task +def setup(): + """ + Setup a fresh virtualenv as well as a few useful directories. + virtualenv should be already installed. + """ + require('hosts', 'app_path', 'virtualenv') + + run('mkdir -p %(app_path)s' % env, pty=True) + run('%(virtualenv)s %(app_path)s/ve' % env, pty=True) + run('mkdir -p %(app_path)s/releases %(app_path)s/packages' % env, pty=True) + run('cd %(app_path)s/releases; ln -s . current; ln -s . previous' % env, pty=True) + upload_samples() + print "Fill out db details in localsettings.py and run deploy." + + +@task(default=True) +def deploy(): + """ + Deploy the latest version of the site to the servers, + install any required third party modules, + install the virtual host and then restart the webserver + """ + require('hosts', 'app_path') + + import time + env.release = time.strftime('%Y-%m-%dT%H%M') + + check_setup() + upload_tar_from_git() + install_requirements() + copy_localsettings() + symlink_current_release() + migrate() + collectstatic() + restart() + +@task +def rollback(): + """ + Limited rollback capability. Simple loads the previously current + version of the code. Rolling back again will swap between the two. + Warning: this will almost certainly go wrong, it there were any migrations + in the meantime! + """ + require('hosts', 'app_path') + with cd(env.path): + run('mv releases/current releases/_previous;', pty=True) + run('mv releases/previous releases/current;', pty=True) + run('mv releases/_previous releases/previous;', pty=True) + collectstatic() + restart() + +@task +def deploy_version(version): + """ + Loads the specified version. + Warning: this will almost certainly go wrong, it there were any migrations + in the meantime! + """ + "Specify a specific version to be made live" + require('hosts', 'app_path') + env.version = version + with cd(env.path): + run('rm releases/previous; mv releases/current releases/previous;', pty=True) + run('ln -s %(version)s releases/current' % env, pty=True) + collectstatic() + restart() + +@task +def restart(): + require('services') + for service in env.services: + execute(service) + + +# ===================================================================== +# = Helpers. These are called by other functions rather than directly = +# ===================================================================== +class DebianGunicorn(Task): + def __init__(self, name): + super(Task, self).__init__() + self.name = name + + def run(self): + print '>>> restart webserver using gunicorn-debian' + with path('/sbin'): + sudo('gunicorn-debian restart %s' % self.site_name, shell=False) + +class Apache(Task): + def run(self): + print '>>> restart webserver by touching WSGI' + with path('/sbin'): + run('touch %(app_path)s/%(project_name)s/wsgi.py' % env) + +class Supervisord(Task): + def __init__(self, name): + super(Task, self).__init__() + self.name = name + + def run(self): + print '>>> supervisord: restart %s' % self.name + with path('/sbin'): + sudo('supervisorctl restart %s' % self.name, shell=False) + +def check_setup(): + require('app_path') + try: + run('[ -e %(app_path)s/ve ]' % env) + except SystemExit: + print "Environment isn't ready. Run fab deploy.setup first." + raise + +def upload_samples(): + upload_localsettings_sample() + upload_nginx_sample() + upload_gunicorn_sample() + +def upload_localsettings_sample(): + "Fill out localsettings template and upload as a sample." + print '>>> upload localsettings template' + require('app_path', 'project_name') + template = '%(project_name)s/localsettings.py.template' + if not exists(template): + template = join(dirname(abspath(__file__)), 'localsettings.py.template') + env.secret_key = '' # sth random + files.upload_template(template, '%(app_path)s/localsettings.py.sample' % env, env) + +def upload_nginx_sample(): + "Fill out nginx conf template and upload as a sample." + print '>>> upload nginx template' + require('app_path', 'project_name') + template = '%(project_name)s/nginx.template' + if not exists(template): + template = join(dirname(abspath(__file__)), 'nginx.template') + files.upload_template(template, '%(app_path)s/nginx.sample' % env, env) + +def upload_gunicorn_sample(): + "Fill out gunicorn conf template and upload as a sample." + print '>>> upload gunicorn template' + require('app_path', 'project_name') + template = '%(project_name)s/gunicorn.template' + if not exists(template): + template = join(dirname(abspath(__file__)), 'gunicorn.template') + files.upload_template(template, '%(app_path)s/gunicorn.sample' % env, env) + +def upload_tar_from_git(): + "Create an archive from the current Git branch and upload it" + print '>>> upload tar from git' + require('release', provided_by=[deploy]) + require('app_path') + local('git-archive-all.sh --format tar %(release)s.tar' % env) + local('gzip %(release)s.tar' % env) + run('mkdir -p %(app_path)s/releases/%(release)s' % env, pty=True) + run('mkdir -p %(app_path)s/packages' % env, pty=True) + put('%(release)s.tar.gz' % env, '%(app_path)s/packages/' % env) + run('cd %(app_path)s/releases/%(release)s && tar zxf ../../packages/%(release)s.tar.gz' % env, pty=True) + local('rm %(release)s.tar.gz' % env) + +def install_requirements(): + "Install the required packages from the requirements file using pip" + print '>>> install requirements' + require('release', provided_by=[deploy]) + require('app_path') + run('cd %(app_path)s; ve/bin/pip install -r %(app_path)s/releases/%(release)s/requirements.txt' % env, pty=True) + +def copy_localsettings(): + "Copy localsettings.py from root directory to release directory (if this file exists)" + print ">>> copy localsettings" + require('release', provided_by=[deploy]) + require('app_path', 'project_name') + + with settings(warn_only=True): + run('cp %(app_path)s/localsettings.py %(app_path)s/releases/%(release)s/%(project_name)s' % env) + +def symlink_current_release(): + "Symlink our current release" + print '>>> symlink current release' + require('release', provided_by=[deploy]) + require('app_path') + with cd(env.path): + run('rm releases/previous; mv releases/current releases/previous') + run('ln -s %(release)s releases/current' % env) + +def migrate(): + "Update the database" + print '>>> migrate' + require('app_path', 'project_name') + with cd('%(app_path)s/releases/current/%(project_name)s' % env): + run('%(app_path)s/ve/bin/python manage.py syncdb --noinput' % env, pty=True) + run('%(app_path)s/ve/bin/python manage.py migrate' % env, pty=True) + +def collectstatic(): + """Collect static files""" + print '>>> collectstatic' + require('app_path', 'project_name') + with cd('%(app_path)s/releases/current/%(project_name)s' % env): + run('%(app_path)s/ve/bin/python manage.py collectstatic --noinput' % env, pty=True) diff --git a/fnpdjango/deploy/templates/gunicorn.template b/fnpdjango/deploy/templates/gunicorn.template new file mode 100644 index 0000000..f79aa7e --- /dev/null +++ b/fnpdjango/deploy/templates/gunicorn.template @@ -0,0 +1,14 @@ +CONFIG = { + 'mode': 'wsgi', + 'working_dir': '%(app_path)s/releases/current/%(project_name)s', + 'python': '%(app_path)s/ve/bin/python', + 'user': '%(user)s', + 'group': '%(user)s', + 'args': ( + '--bind=unix:/tmp/gunicorn-%(project_name)s.socket', + '--workers=1', + '--timeout=180', + 'wsgi:application', + '--access-logfile=%(app_path)s/log/gunicorn.log', + ), +} diff --git a/fnpdjango/deploy/templates/localsettings.py.template b/fnpdjango/deploy/templates/localsettings.py.template new file mode 100644 index 0000000..551e00f --- /dev/null +++ b/fnpdjango/deploy/templates/localsettings.py.template @@ -0,0 +1,24 @@ +ADMINS = ( + ('Lista IT', 'it@listy.nowoczesnapolska.org.pl'), +) +MANAGERS = ( + ('Lista IT', 'it@listy.nowoczesnapolska.org.pl'), +) + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'NAME': '%(project_name)s', # Or path to database file if using sqlite3. + 'USER': '%(project_name)s', # Not used with sqlite3. + 'PASSWORD': '', # Not used with sqlite3. + 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '', # Set to empty string for default. Not used with sqlite3. + } +} + +SECRET_KEY = '%(secret_key)s' +PIWIK_URL = 'http://piwik.nowoczesnapolska.org.pl/' +PIWIK_TOKEN = '' + +MEDIA_ROOT = '%(app_path)s/media/' +STATIC_ROOT = '%(app_path)s/static/' diff --git a/fnpdjango/deploy/templates/nginx.template b/fnpdjango/deploy/templates/nginx.template new file mode 100644 index 0000000..810c73c --- /dev/null +++ b/fnpdjango/deploy/templates/nginx.template @@ -0,0 +1,17 @@ +server { + listen [::]:80; + listen 80; + server_name %(project_name)s.nowoczesnapolska.org.pl?; + access_log %(app_path)s/log/nginx.log; + error_log %(app_path)s/log/nginx-error.log; + + root %(path)s; + location /media/ { + } + location /static/ { + } + location / { + include /etc/nginx/proxy.conf; + proxy_pass http://unix:/tmp/gunicorn-%(project_name)s.socket; + } +} diff --git a/fnpdjango/templatetags/macros.py b/fnpdjango/templatetags/macros.py new file mode 100755 index 0000000..181a497 --- /dev/null +++ b/fnpdjango/templatetags/macros.py @@ -0,0 +1,160 @@ +# +# templatetags/macros.py - Support for macros in Django templates +# +# Author: Michal Ludvig +# http://www.logix.cz/michal +# + +""" +Tag library that provides support for "macros" in +Django templates. + +Usage example: + +0) Save this file as + /taglibrary/macros.py + +1) In your template load the library: + {% load macros %} + +2) Define a new macro called 'my_macro' with + parameter 'arg1': + {% macro my_macro arg1 %} + Parameter: {{ arg1 }}
+ {% endmacro %} + +3) Use the macro with a String parameter: + {% usemacro my_macro "String parameter" %} + + or with a variable parameter (provided the + context defines 'somearg' variable, e.g. with + value "Variable parameter"): + {% usemacro my_macro somearg %} + + The output of the above code would be: + Parameter: String parameter
+ Parameter: Variable parameter
+ +4) Alternatively save your macros in a separate + file, e.g. "mymacros.html" and load it to the + current template with: + {% loadmacros "mymacros.html" %} + Then use these loaded macros in {% usemacro %} + as described above. + +Macros can take zero or more arguments and both +context variables and macro arguments are resolved +in macro body when used in {% usemacro ... %} tag. + +Bear in mind that defined and loaded Macros are local +to each template file and are not inherited +through {% extends ... %} tags. +""" + +from django import template +from django.template import resolve_variable, FilterExpression +from django.template.loader import get_template, get_template_from_string, find_template_source +from django.conf import settings +import re + +register = template.Library() + +def _setup_macros_dict(parser): + ## Metadata of each macro are stored in a new attribute + ## of 'parser' class. That way we can access it later + ## in the template when processing 'usemacro' tags. + try: + ## Only try to access it to eventually trigger an exception + parser._macros + except AttributeError: + parser._macros = {} + +class DefineMacroNode(template.Node): + def __init__(self, name, nodelist, args): + self.name = name + self.nodelist = nodelist + self.args = args + + def render(self, context): + ## empty string - {% macro %} tag does no output + return '' + +@register.tag(name="macro") +def do_macro(parser, token): + try: + args = token.split_contents() + tag_name, macro_name, args = args[0], args[1], args[2:] + except IndexError: + raise template.TemplateSyntaxError, "'%s' tag requires at least one argument (macro name)" % token.contents.split()[0] + # TODO: check that 'args' are all simple strings ([a-zA-Z0-9_]+) + r_valid_arg_name = re.compile(r'^[a-zA-Z0-9_]+$') + for arg in args: + if not r_valid_arg_name.match(arg): + raise template.TemplateSyntaxError, "Argument '%s' to macro '%s' contains illegal characters. Only alphanumeric characters and '_' are allowed." % (arg, macro_name) + nodelist = parser.parse(('endmacro', )) + parser.delete_first_token() + + ## Metadata of each macro are stored in a new attribute + ## of 'parser' class. That way we can access it later + ## in the template when processing 'usemacro' tags. + _setup_macros_dict(parser) + + parser._macros[macro_name] = DefineMacroNode(macro_name, nodelist, args) + return parser._macros[macro_name] + +class LoadMacrosNode(template.Node): + def render(self, context): + ## empty string - {% loadmacros %} tag does no output + return '' + +@register.tag(name="loadmacros") +def do_loadmacros(parser, token): + try: + tag_name, filename = token.split_contents() + except IndexError: + raise template.TemplateSyntaxError, "'%s' tag requires at least one argument (macro name)" % token.contents.split()[0] + if filename[0] in ('"', "'") and filename[-1] == filename[0]: + filename = filename[1:-1] + t = get_template(filename) + macros = t.nodelist.get_nodes_by_type(DefineMacroNode) + ## Metadata of each macro are stored in a new attribute + ## of 'parser' class. That way we can access it later + ## in the template when processing 'usemacro' tags. + _setup_macros_dict(parser) + for macro in macros: + parser._macros[macro.name] = macro + return LoadMacrosNode() + +class UseMacroNode(template.Node): + def __init__(self, macro, filter_expressions): + self.nodelist = macro.nodelist + self.args = macro.args + self.filter_expressions = filter_expressions + def render(self, context): + for (arg, fe) in [(self.args[i], self.filter_expressions[i]) for i in range(len(self.args))]: + context[arg] = fe.resolve(context) + return self.nodelist.render(context) + +@register.tag(name="usemacro") +def do_usemacro(parser, token): + try: + args = token.split_contents() + tag_name, macro_name, values = args[0], args[1], args[2:] + except IndexError: + raise template.TemplateSyntaxError, "'%s' tag requires at least one argument (macro name)" % token.contents.split()[0] + try: + macro = parser._macros[macro_name] + except (AttributeError, KeyError): + raise template.TemplateSyntaxError, "Macro '%s' is not defined" % macro_name + + if (len(values) != len(macro.args)): + raise template.TemplateSyntaxError, "Macro '%s' was declared with %d parameters and used with %d parameter" % ( + macro_name, + len(macro.args), + len(values)) + filter_expressions = [] + for val in values: + if (val[0] == "'" or val[0] == '"') and (val[0] != val[-1]): + raise template.TemplateSyntaxError, "Non-terminated string argument: %s" % val[1:] + filter_expressions.append(FilterExpression(val, parser)) + return UseMacroNode(macro, filter_expressions) \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..108a2ec --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +fabric>=1.5,<1.6 diff --git a/setup.py b/setup.py index ecf8f8f..5020a9b 100755 --- a/setup.py +++ b/setup.py @@ -27,8 +27,18 @@ setup( author_email='radoslaw.czajka@nowoczesnapolska.org.pl', url = '', packages=find_packages(), - package_data={'fnpdjango': whole_trees('fnpdjango', - ['templates', 'locale'])}, + package_data={'fnpdjango': + whole_trees('fnpdjango', ['templates', 'locale']) + + whole_trees('fnpdjango/deploy', ['templates']) + }, + scripts=[ + 'bin/git-archive-all.sh', + 'bin/fnpdjango_bootstrap.sh', + ], + install_requires=[ + 'django>=1.4,<1.5', + 'textile', + ], license='LICENSE', description='.', long_description="",