add utils.pipeline_storage.GzipPipelineCachedStorage

[fnpdjango.git] / fnpdjango / deploy / __init__.py

diff --git a/fnpdjango/deploy/__init__.py b/fnpdjango/deploy/__init__.py
index 9c7d762..eaf5e71 100644 (file)
--- a/fnpdjango/deploy/__init__.py
+++ b/fnpdjango/deploy/__init__.py
@@ -10,10 +10,18 @@ Then set up some env properties:
     user: remote user name
     app_path: where does the app go
     services: list of tasks to run after deployment
     user: remote user name
     app_path: where does the app go
     services: list of tasks to run after deployment

-
+    django_root_path (optional): path to the directory
+        containing django project, relative to the
+        root of the repository (defaults to '.')
+    localsettings_dst_path (optional): path indicating
+        where to copy the localsettings file, relative
+        to django_root_path (defaults to project_name/localsettings.py)
+    skip_collect_static (optional): if True, Django collectstatic command is not called

 """
 """

-from fabric.api import *
+from subprocess import check_output

 from os.path import abspath, dirname, exists, join
 from os.path import abspath, dirname, exists, join

+from django.utils.crypto import get_random_string
+from fabric.api import *

 from fabric.contrib import files
 from fabric.tasks import Task, execute
 
 from fabric.contrib import files
 from fabric.tasks import Task, execute
 


@@ -24,17 +32,27 @@ env.services = None
 @task
 def setup():
     """
 @task
 def setup():
     """

-    Setup a fresh virtualenv as well as a few useful directories.
-    virtualenv should be already installed.
+    Setup all needed directories.

     """
     """

-    require('hosts', 'app_path', 'virtualenv')
+    require('hosts', 'app_path')

 
 

-    run('mkdir -p %(app_path)s' % env, pty=True)
-    run('%(virtualenv)s %(app_path)s/ve' % env, pty=True)
-    run('mkdir -p %(app_path)s/releases %(app_path)s/packages' % env, pty=True)
-    run('cd %(app_path)s/releases; ln -s . current; ln -s . previous' % env, pty=True)
+    if not files.exists(env.app_path):
+        run('mkdir -p %(app_path)s' % env, pty=True)
+    with cd(env.app_path):
+        for subdir in 'releases', 'packages', 'log', 'samples':
+            if not files.exists(subdir):
+                run('mkdir -p %s' % subdir, pty=True)
+    with cd('%(app_path)s/releases' % env):
+        if not files.exists('current'):
+            run('ln -sfT . current', pty=True)
+        if not files.exists('previous'):
+            run('ln -sfT . previous', pty=True)
+    

     upload_samples()
     upload_samples()

-    print "Fill out db details in localsettings.py and run deploy."
+
+
+def check_localsettings():
+    return files.exists('%(app_path)s/localsettings.py' % env)

 
 
 @task(default=True)
 
 
 @task(default=True)


@@ -47,14 +65,18 @@ def deploy():
     require('hosts', 'app_path')
 
     import time
     require('hosts', 'app_path')
 
     import time

-    env.release = time.strftime('%Y-%m-%dT%H%M')
+    env.release = '%s_%s' % (time.strftime('%Y-%m-%dT%H%M'), check_output(['git', 'rev-parse', 'HEAD']).strip())

 
 

-    check_setup()
+    setup()
+    if not check_localsettings():
+        abort('Setup is complete, but\n    %(app_path)s/localsettings.py\n'
+              'is needed for actual deployment.' % env)

     upload_tar_from_git()
     upload_tar_from_git()

-    install_requirements()

     copy_localsettings()
     copy_localsettings()

+    install_requirements()

     symlink_current_release()
     migrate()
     symlink_current_release()
     migrate()

+    pre_collectstatic()

     collectstatic()
     restart()
 
     collectstatic()
     restart()
 


@@ -92,80 +114,84 @@ def deploy_version(version):
 
 @task
 def restart():
 
 @task
 def restart():

-    require('services')
-    for service in env.services:
+    for service in env.services or ():

         execute(service)
 
 
 # =====================================================================
 # = Helpers. These are called by other functions rather than directly =
 # =====================================================================
         execute(service)
 
 
 # =====================================================================
 # = Helpers. These are called by other functions rather than directly =
 # =====================================================================

-class DebianGunicorn(Task):
+
+class Service(Task):
+    def upload_sample(self):
+        pass
+
+class DebianGunicorn(Service):

     def __init__(self, name):
         super(Task, self).__init__()
         self.name = name
 
     def run(self):
         print '>>> restart webserver using gunicorn-debian'
     def __init__(self, name):
         super(Task, self).__init__()
         self.name = name
 
     def run(self):
         print '>>> restart webserver using gunicorn-debian'

-        with path('/sbin'):
-            sudo('gunicorn-debian restart %s' % self.site_name, shell=False)
+        sudo('gunicorn-debian restart %s' % self.name, shell=False)

 
 

-class Apache(Task):
+    def upload_sample(self):
+        upload_sample('gunicorn', additional_context = dict(django_root_path = get_django_root_path(env['release'])))
+
+class Apache(Service):

     def run(self):
         print '>>> restart webserver by touching WSGI'
         with path('/sbin'):
             run('touch %(app_path)s/%(project_name)s/wsgi.py' % env)
 
     def run(self):
         print '>>> restart webserver by touching WSGI'
         with path('/sbin'):
             run('touch %(app_path)s/%(project_name)s/wsgi.py' % env)
 

-class Supervisord(Task):
+class Supervisord(Service):

     def __init__(self, name):
         super(Task, self).__init__()
         self.name = name
 
     def run(self):
         print '>>> supervisord: restart %s' % self.name
     def __init__(self, name):
         super(Task, self).__init__()
         self.name = name
 
     def run(self):
         print '>>> supervisord: restart %s' % self.name

-        with path('/sbin'):
-            sudo('supervisorctl restart %s' % self.name, shell=False)
+        sudo('supervisorctl restart %s' % self.name, shell=False)

 
 

-def check_setup():
-    require('app_path')
-    try:
-        run('[ -e %(app_path)s/ve ]' % env)
-    except SystemExit:
-        print "Environment isn't ready. Run fab deploy.setup first."
-        raise
+class Command(Task):
+    def __init__(self, commands, working_dir):
+        if not hasattr(commands, '__iter__'):
+            commands = [commands]
+        self.name = 'Command: %s @ %s' % (commands, working_dir)
+        self.commands = commands
+        self.working_dir = working_dir
+
+    def run(self):
+        require('app_path')
+        with cd(join('%(app_path)s/releases/current' % env, self.working_dir)):
+            for command in self.commands:
+                run(command)

 
 def upload_samples():
     upload_localsettings_sample()
     upload_nginx_sample()
 
 def upload_samples():
     upload_localsettings_sample()
     upload_nginx_sample()

-    upload_gunicorn_sample()
+    for service in env.services:
+        service.upload_sample()

 
 

-def upload_localsettings_sample():
-    "Fill out localsettings template and upload as a sample."
-    print '>>> upload localsettings template'
+def upload_sample(name, where="samples/", additional_context=None):

     require('app_path', 'project_name')
     require('app_path', 'project_name')

-    template = '%(project_name)s/localsettings.py.template'
+    upload_path = '%s/%s%s.sample' % (env['app_path'], where, name)
+    if files.exists(upload_path):
+        return
+    print '>>> upload %s template' % name
+    template = '%(project_name)s/' % env + name + '.template'

     if not exists(template):
     if not exists(template):

-        template = join(dirname(abspath(__file__)), 'localsettings.py.template')
-    env.secret_key = '' # sth random
-    files.upload_template(template, '%(app_path)s/localsettings.py.sample' % env, env)
+        template = join(dirname(abspath(__file__)), 'templates/' + name + '.template')
+    template_context = additional_context or dict()
+    template_context.update(env)
+    files.upload_template(template, upload_path, template_context)

 
 

-def upload_nginx_sample():
-    "Fill out nginx conf template and upload as a sample."
-    print '>>> upload nginx template'
-    require('app_path', 'project_name')
-    template = '%(project_name)s/nginx.template'
-    if not exists(template):
-        template = join(dirname(abspath(__file__)), 'nginx.template')
-    files.upload_template(template, '%(app_path)s/nginx.sample' % env, env)
+def upload_localsettings_sample():
+    "Fill out localsettings template and upload as a sample."
+    env.secret_key = get_random_string(50)
+    upload_sample('localsettings.py', where="")

 
 

-def upload_gunicorn_sample():
-    "Fill out gunicorn conf template and upload as a sample."
-    print '>>> upload gunicorn template'
-    require('app_path', 'project_name')
-    template = '%(project_name)s/gunicorn.template'
-    if not exists(template):
-        template = join(dirname(abspath(__file__)), 'gunicorn.template')
-    files.upload_template(template, '%(app_path)s/gunicorn.sample' % env, env)
+upload_nginx_sample = lambda: upload_sample('nginx')

 
 def upload_tar_from_git():
     "Create an archive from the current Git branch and upload it"
 
 def upload_tar_from_git():
     "Create an archive from the current Git branch and upload it"


@@ -185,7 +211,23 @@ def install_requirements():
     print '>>> install requirements'
     require('release', provided_by=[deploy])
     require('app_path')
     print '>>> install requirements'
     require('release', provided_by=[deploy])
     require('app_path')

-    run('cd %(app_path)s; ve/bin/pip install -r %(app_path)s/releases/%(release)s/requirements.txt' % env, pty=True)
+    if not files.exists('%(app_path)s/ve' % env):
+        require('virtualenv')
+        run('%(virtualenv)s %(app_path)s/ve' % env, pty=True)
+    with cd('%(app_path)s/releases/%(release)s' % env):
+        run('%(app_path)s/ve/bin/pip install -r requirements.txt' % env, pty=True)
+    with cd(get_django_root_path(env['release'])):
+        # Install DB requirement
+        database_reqs = {
+            'django.db.backends.postgresql_psycopg2': 'psycopg2',
+            'django.db.backends.mysql': 'MySQL-python',
+        }
+        databases = run('''DJANGO_SETTINGS_MODULE=%(project_name)s.settings %(app_path)s/ve/bin/python -c 'from django.conf import settings;              print " ".join(set([d["ENGINE"] for d in settings.DATABASES.values()]))' ''' % env)
+        for database in databases.split():
+            if database in database_reqs:
+                # TODO: set pip default pypi
+                run('%(app_path)s/ve/bin/pip install ' % env + database_reqs[database])
+

 
 def copy_localsettings():
     "Copy localsettings.py from root directory to release directory (if this file exists)"
 
 def copy_localsettings():
     "Copy localsettings.py from root directory to release directory (if this file exists)"


@@ -194,14 +236,15 @@ def copy_localsettings():
     require('app_path', 'project_name')
 
     with settings(warn_only=True):
     require('app_path', 'project_name')
 
     with settings(warn_only=True):

-        run('cp %(app_path)s/localsettings.py %(app_path)s/releases/%(release)s/%(project_name)s' % env)
+        copy_to = join(get_django_root_path(env['release']), env.get('localsettings_dst_path', env['project_name']))
+        run('cp %(app_path)s/localsettings.py ' % env + copy_to)

 
 def symlink_current_release():
     "Symlink our current release"
     print '>>> symlink current release'
     require('release', provided_by=[deploy])
     require('app_path')
 
 def symlink_current_release():
     "Symlink our current release"
     print '>>> symlink current release'
     require('release', provided_by=[deploy])
     require('app_path')

-    with cd(env.path):
+    with cd(env.app_path):

         run('rm releases/previous; mv releases/current releases/previous')
         run('ln -s %(release)s releases/current' % env)
 
         run('rm releases/previous; mv releases/current releases/previous')
         run('ln -s %(release)s releases/current' % env)
 


@@ -209,13 +252,29 @@ def migrate():
     "Update the database"
     print '>>> migrate'
     require('app_path', 'project_name')
     "Update the database"
     print '>>> migrate'
     require('app_path', 'project_name')

-    with cd('%(app_path)s/releases/current/%(project_name)s' % env):
+    with cd(get_django_root_path('current')):

         run('%(app_path)s/ve/bin/python manage.py syncdb --noinput' % env, pty=True)
         run('%(app_path)s/ve/bin/python manage.py migrate' % env, pty=True)
 
         run('%(app_path)s/ve/bin/python manage.py syncdb --noinput' % env, pty=True)
         run('%(app_path)s/ve/bin/python manage.py migrate' % env, pty=True)
 

+def pre_collectstatic():
+    print '>>> pre_collectstatic'
+    for task in env.get('pre_collectstatic', []):
+        execute(task)
+

 def collectstatic():
     """Collect static files"""
     print '>>> collectstatic'
 def collectstatic():
     """Collect static files"""
     print '>>> collectstatic'

+    if env.get('skip_collect_static', False):
+        print '... skipped'
+        return

     require('app_path', 'project_name')
     require('app_path', 'project_name')

-    with cd('%(app_path)s/releases/current/%(project_name)s' % env):
+    with cd(get_django_root_path('current')):

         run('%(app_path)s/ve/bin/python manage.py collectstatic --noinput' % env, pty=True)
         run('%(app_path)s/ve/bin/python manage.py collectstatic --noinput' % env, pty=True)

+
+
+def get_django_root_path(release):
+    require('app_path')
+    path = '%(app_path)s/releases/%(release)s' % dict(app_path = env['app_path'], release = release)
+    if 'django_root_path' in env:
+        path = join(path, env['django_root_path'])
+    return path