apps/south/migration.py

   1
   2 import datetime
   3 import os
   4 import sys
   5 from django.conf import settings
   6 from django.db import models
   7 from models import MigrationHistory
   8 from south.db import db
   9
  10
  11 def get_app(app):
  12     """
  13     Returns the migrations module for the given app model name/module, or None
  14     if it does not use migrations.
  15     """
  16     if isinstance(app, (str, unicode)):
  17         # If it's a string, use the models module
  18         app = models.get_app(app)
  19     mod = __import__(app.__name__[:-7], {}, {}, ['migrations'])
  20     if hasattr(mod, 'migrations'):
  21         return getattr(mod, 'migrations')
  22
  23
  24 def get_migrated_apps():
  25     """
  26     Returns all apps with migrations.
  27     """
  28     for mapp in models.get_apps():
  29         app = get_app(mapp)
  30         if app:
  31             yield app
  32
  33
  34 def get_app_name(app):
  35     """
  36     Returns the _internal_ app name for the given app module.
  37     i.e. for <module django.contrib.auth.models> will return 'auth'
  38     """
  39     return app.__name__.split('.')[-2]
  40
  41
  42 def get_app_fullname(app):
  43     """
  44     Returns the full python name of an app - e.g. django.contrib.auth
  45     """
  46     return app.__name__[:-11]
  47
  48
  49 def short_from_long(app_name):
  50     return app_name.split(".")[-1]
  51
  52
  53 def get_migration_names(app):
  54     """
  55     Returns a list of migration file names for the given app.
  56     """
  57     return sorted([
  58         filename[:-3]
  59         for filename in os.listdir(os.path.dirname(app.__file__))
  60         if filename.endswith(".py") and filename != "__init__.py"
  61     ])
  62
  63
  64 def get_migration_classes(app):
  65     """
  66     Returns a list of migration classes (one for each migration) for the app.
  67     """
  68     for name in get_migration_names(app):
  69         yield get_migration(app, name)
  70
  71
  72 def get_migration(app, name):
  73     """
  74     Returns the migration class implied by 'name'.
  75     """
  76     try:
  77         module = __import__(app.__name__ + "." + name, '', '', ['Migration'])
  78         return module.Migration
  79     except ImportError:
  80         raise ValueError("Migration %s:%s does not exist." % (get_app_name(app), name))
  81
  82
  83 def all_migrations():
  84     return dict([
  85         (app, dict([(name, get_migration(app, name)) for name in get_migration_names(app)]))
  86         for app in get_migrated_apps()
  87     ])
  88
  89
  90 def dependency_tree():
  91     tree = all_migrations()
  92
  93     # Annotate tree with 'backwards edges'
  94     for app, classes in tree.items():
  95         for name, cls in classes.items():
  96             cls.needs = []
  97             if not hasattr(cls, "needed_by"):
  98                 cls.needed_by = []
  99             if hasattr(cls, "depends_on"):
 100                 for dapp, dname in cls.depends_on:
 101                     dapp = get_app(dapp)
 102                     if dapp not in tree:
 103                         print "Migration %s in app %s depends on unmigrated app %s." % (
 104                             name,
 105                             get_app_name(app),
 106                             dapp,
 107                         )
 108                         sys.exit(1)
 109                     if dname not in tree[dapp]:
 110                         print "Migration %s in app %s depends on nonexistent migration %s in app %s." % (
 111                             name,
 112                             get_app_name(app),
 113                             dname,
 114                             get_app_name(dapp),
 115                         )
 116                         sys.exit(1)
 117                     cls.needs.append((dapp, dname))
 118                     if not hasattr(tree[dapp][dname], "needed_by"):
 119                         tree[dapp][dname].needed_by = []
 120                     tree[dapp][dname].needed_by.append((app, name))
 121
 122     # Sanity check whole tree
 123     for app, classes in tree.items():
 124         for name, cls in classes.items():
 125             cls.dependencies = dependencies(tree, app, name)
 126
 127     return tree
 128
 129
 130 def nice_trace(trace):
 131     return " -> ".join([str((get_app_name(a), n)) for a, n in trace])
 132
 133
 134 def dependencies(tree, app, name, trace=[]):
 135     # Copy trace to stop pass-by-ref problems
 136     trace = trace[:]
 137     # Sanity check
 138     for papp, pname in trace:
 139         if app == papp:
 140             if pname == name:
 141                 print "Found circular dependency: %s" % nice_trace(trace + [(app,name)])
 142                 sys.exit(1)
 143             else:
 144                 # See if they depend in the same app the wrong way
 145                 migrations = get_migration_names(app)
 146                 if migrations.index(name) > migrations.index(pname):
 147                     print "Found a lower migration (%s) depending on a higher migration (%s) in the same app (%s)." % (pname, name, get_app_name(app))
 148                     print "Path: %s" % nice_trace(trace + [(app,name)])
 149                     sys.exit(1)
 150     # Get the dependencies of a migration
 151     deps = []
 152     migration = tree[app][name]
 153     for dapp, dname in migration.needs:
 154         deps.extend(
 155             dependencies(tree, dapp, dname, trace+[(app,name)])
 156         )
 157     return deps
 158
 159
 160 def remove_duplicates(l):
 161     m = []
 162     for x in l:
 163         if x not in m:
 164             m.append(x)
 165     return m
 166
 167
 168 def needed_before_forwards(tree, app, name, sameapp=True):
 169     """
 170     Returns a list of migrations that must be applied before (app, name),
 171     in the order they should be applied.
 172     Used to make sure a migration can be applied (and to help apply up to it).
 173     """
 174     app_migrations = get_migration_names(app)
 175     needed = []
 176     if sameapp:
 177         for aname in app_migrations[:app_migrations.index(name)]:
 178             needed += needed_before_forwards(tree, app, aname, False)
 179             needed += [(app, aname)]
 180     for dapp, dname in tree[app][name].needs:
 181         needed += needed_before_forwards(tree, dapp, dname)
 182         needed += [(dapp, dname)]
 183     return remove_duplicates(needed)
 184
 185
 186 def needed_before_backwards(tree, app, name, sameapp=True):
 187     """
 188     Returns a list of migrations that must be unapplied before (app, name) is,
 189     in the order they should be unapplied.
 190     Used to make sure a migration can be unapplied (and to help unapply up to it).
 191     """
 192     app_migrations = get_migration_names(app)
 193     needed = []
 194     if sameapp:
 195         for aname in reversed(app_migrations[app_migrations.index(name)+1:]):
 196             needed += needed_before_backwards(tree, app, aname, False)
 197             needed += [(app, aname)]
 198     for dapp, dname in tree[app][name].needed_by:
 199         needed += needed_before_backwards(tree, dapp, dname)
 200         needed += [(dapp, dname)]
 201     return remove_duplicates(needed)
 202
 203
 204 def run_forwards(app, migrations, fake=False, silent=False):
 205     """
 206     Runs the specified migrations forwards, in order.
 207     """
 208     for migration in migrations:
 209         app_name = get_app_name(app)
 210         if not silent:
 211             print " > %s: %s" % (app_name, migration)
 212         klass = get_migration(app, migration)
 213         if fake:
 214             if not silent:
 215                 print "   (faked)"
 216         else:
 217             db.start_transaction()
 218             try:
 219                 klass().forwards()
 220                 db.execute_deferred_sql()
 221             except:
 222                 db.rollback_transaction()
 223                 raise
 224             else:
 225                 db.commit_transaction()
 226         # Record us as having done this
 227         record = MigrationHistory.for_migration(app_name, migration)
 228         record.applied = datetime.datetime.utcnow()
 229         record.save()
 230
 231
 232 def run_backwards(app, migrations, ignore=[], fake=False, silent=False):
 233     """
 234     Runs the specified migrations backwards, in order, skipping those
 235     migrations in 'ignore'.
 236     """
 237     for migration in migrations:
 238         if migration not in ignore:
 239             app_name = get_app_name(app)
 240             if not silent:
 241                 print " < %s: %s" % (app_name, migration)
 242             klass = get_migration(app, migration)
 243             if fake:
 244                 if not silent:
 245                     print "   (faked)"
 246             else:
 247                 db.start_transaction()
 248                 try:
 249                     klass().backwards()
 250                     db.execute_deferred_sql()
 251                 except:
 252                     db.rollback_transaction()
 253                     raise
 254                 else:
 255                     db.commit_transaction()
 256             # Record us as having not done this
 257             record = MigrationHistory.for_migration(app_name, migration)
 258             record.delete()
 259
 260
 261 def right_side_of(x, y):
 262     return left_side_of(reversed(x), reversed(y))
 263
 264
 265 def left_side_of(x, y):
 266     return list(y)[:len(x)] == list(x)
 267
 268
 269 def forwards_problems(tree, forwards, done, silent=False):
 270     problems = []
 271     for app, name in forwards:
 272         if (app, name) not in done:
 273             for dapp, dname in needed_before_backwards(tree, app, name):
 274                 if (dapp, dname) in done:
 275                     if not silent:
 276                         print " ! Migration (%s, %s) should not have been applied before (%s, %s) but was." % (get_app_name(dapp), dname, get_app_name(app), name)
 277                     problems.append(((app, name), (dapp, dname)))
 278     return problems
 279
 280
 281
 282 def backwards_problems(tree, backwards, done, silent=False):
 283     problems = []
 284     for app, name in backwards:
 285         if (app, name) in done:
 286             for dapp, dname in needed_before_forwards(tree, app, name):
 287                 if (dapp, dname) not in done:
 288                     if not silent:
 289                         print " ! Migration (%s, %s) should have been applied before (%s, %s) but wasn't." % (get_app_name(dapp), dname, get_app_name(app), name)
 290                     problems.append(((app, name), (dapp, dname)))
 291     return problems
 292
 293
 294 def migrate_app(app, target_name=None, resolve_mode=None, fake=False, yes=False, silent=False):
 295
 296     app_name = get_app_name(app)
 297
 298     db.debug = not silent
 299
 300     # If any of their app names in the DB contain a ., they're 0.2 or below, so migrate em
 301     longuns = MigrationHistory.objects.filter(app_name__contains=".")
 302     if longuns:
 303         for mh in longuns:
 304             mh.app_name = short_from_long(mh.app_name)
 305             mh.save()
 306         if not silent:
 307             print "- Updated your South 0.2 database."
 308
 309     # Find out what delightful migrations we have
 310     tree = dependency_tree()
 311     migrations = get_migration_names(app)
 312
 313     if target_name not in migrations and target_name not in ["zero", None]:
 314         matches = [x for x in migrations if x.startswith(target_name)]
 315         if len(matches) == 1:
 316             target = migrations.index(matches[0]) + 1
 317             if not silent:
 318                 print " - Soft matched migration %s to %s." % (
 319                     target_name,
 320                     matches[0]
 321                 )
 322             target_name = matches[0]
 323         elif len(matches) > 1:
 324             if not silent:
 325                 print " - Prefix %s matches more than one migration:" % target_name
 326                 print "     " + "\n     ".join(matches)
 327             return
 328         else:
 329             if not silent:
 330                 print " ! '%s' is not a migration." % target_name
 331             return
 332
 333     # Check there's no strange ones in the database
 334     ghost_migrations = [m for m in MigrationHistory.objects.filter(applied__isnull = False) if get_app(m.app_name) not in tree or m.migration not in tree[get_app(m.app_name)]]
 335     if ghost_migrations:
 336         if not silent:
 337             print " ! These migrations are in the database but not on disk:"
 338             print "   - " + "\n   - ".join(["%s: %s" % (x.app_name, x.migration) for x in ghost_migrations])
 339             print " ! I'm not trusting myself; fix this yourself by fiddling"
 340             print " ! with the south_migrationhistory table."
 341         return
 342
 343     # Say what we're doing
 344     if not silent:
 345         print "Running migrations for %s:" % app_name
 346
 347     # Get the forwards and reverse dependencies for this target
 348     if target_name == None:
 349         target_name = migrations[-1]
 350     if target_name == "zero":
 351         forwards = []
 352         backwards = needed_before_backwards(tree, app, migrations[0]) + [(app, migrations[0])]
 353     else:
 354         forwards = needed_before_forwards(tree, app, target_name) + [(app, target_name)]
 355         # When migrating backwards we want to remove up to and including
 356         # the next migration up in this app (not the next one, that includes other apps)
 357         try:
 358             migration_before_here = migrations[migrations.index(target_name)+1]
 359             backwards = needed_before_backwards(tree, app, migration_before_here) + [(app, migration_before_here)]
 360         except IndexError:
 361             backwards = []
 362
 363     # Get the list of currently applied migrations from the db
 364     current_migrations = [(get_app(m.app_name), m.migration) for m in  MigrationHistory.objects.filter(applied__isnull = False)]
 365
 366     direction = None
 367     bad = False
 368
 369     # Work out the direction
 370     applied_for_this_app = list(MigrationHistory.objects.filter(app_name=app_name, applied__isnull=False).order_by("migration"))
 371     if target_name == "zero":
 372         direction = -1
 373     elif not applied_for_this_app:
 374         direction = 1
 375     elif migrations.index(target_name) > migrations.index(applied_for_this_app[-1].migration):
 376         direction = 1
 377     elif migrations.index(target_name) < migrations.index(applied_for_this_app[-1].migration):
 378         direction = -1
 379     else:
 380         direction = None
 381
 382     # Is the whole forward branch applied?
 383     missing = [step for step in forwards if step not in current_migrations]
 384     # If they're all applied, we only know it's not backwards
 385     if not missing:
 386         direction = None
 387     # If the remaining migrations are strictly a right segment of the forwards
 388     # trace, we just need to go forwards to our target (and check for badness)
 389     else:
 390         problems = forwards_problems(tree, forwards, current_migrations, silent=silent)
 391         if problems:
 392             bad = True
 393         direction = 1
 394
 395     # What about the whole backward trace then?
 396     if not bad:
 397         missing = [step for step in backwards if step not in current_migrations]
 398         # If they're all missing, stick with the forwards decision
 399         if missing == backwards:
 400             pass
 401         # If what's missing is a strict left segment of backwards (i.e.
 402         # all the higher migrations) then we need to go backwards
 403         else:
 404             problems = backwards_problems(tree, backwards, current_migrations, silent=silent)
 405             if problems:
 406                 bad = True
 407             direction = -1
 408
 409     if bad and resolve_mode not in ['merge']:
 410         if not silent:
 411             print " ! Inconsistent migration history"
 412             print " ! The following options are available:"
 413             print "    --merge: will just attempt the migration ignoring any potential dependency conflicts."
 414         sys.exit(1)
 415
 416     if direction == 1:
 417         if not silent:
 418             print " - Migrating forwards to %s." % target_name
 419         for mapp, mname in forwards:
 420             if (mapp, mname) not in current_migrations:
 421                 run_forwards(mapp, [mname], fake=fake, silent=silent)
 422     elif direction == -1:
 423         if not silent:
 424             print " - Migrating backwards to just after %s." % target_name
 425         for mapp, mname in backwards:
 426             if (mapp, mname) in current_migrations:
 427                 run_backwards(mapp, [mname], fake=fake, silent=silent)
 428     else:
 429         if not silent:
 430             print "- Nothing to migrate."