apps/south/migration.py

   1
   2 import datetime
   3 import os
   4 import sys
   5 import traceback
   6 from django.conf import settings
   7 from django.db import models
   8 from django.core.exceptions import ImproperlyConfigured
   9 from django.core.management import call_command
  10 from models import MigrationHistory
  11 from south.db import db
  12
  13
  14 def get_app(app):
  15     """
  16     Returns the migrations module for the given app model name/module, or None
  17     if it does not use migrations.
  18     """
  19     if isinstance(app, (str, unicode)):
  20         # If it's a string, use the models module
  21         app = models.get_app(app)
  22     mod = __import__(app.__name__[:-7], {}, {}, ['migrations'])
  23     if hasattr(mod, 'migrations'):
  24         return getattr(mod, 'migrations')
  25
  26
  27 def get_migrated_apps():
  28     """
  29     Returns all apps with migrations.
  30     """
  31     for mapp in models.get_apps():
  32         app = get_app(mapp)
  33         if app:
  34             yield app
  35
  36
  37 def get_app_name(app):
  38     """
  39     Returns the _internal_ app name for the given app module.
  40     i.e. for <module django.contrib.auth.models> will return 'auth'
  41     """
  42     return app.__name__.split('.')[-2]
  43
  44
  45 def get_app_fullname(app):
  46     """
  47     Returns the full python name of an app - e.g. django.contrib.auth
  48     """
  49     return app.__name__[:-11]
  50
  51
  52 def short_from_long(app_name):
  53     return app_name.split(".")[-1]
  54
  55
  56 def get_migration_names(app):
  57     """
  58     Returns a list of migration file names for the given app.
  59     """
  60     return sorted([
  61         filename[:-3]
  62         for filename in os.listdir(os.path.dirname(app.__file__))
  63         if filename.endswith(".py") and filename != "__init__.py" and not filename.startswith(".")
  64     ])
  65
  66
  67 def get_migration_classes(app):
  68     """
  69     Returns a list of migration classes (one for each migration) for the app.
  70     """
  71     for name in get_migration_names(app):
  72         yield get_migration(app, name)
  73
  74
  75 def get_migration(app, name):
  76     """
  77     Returns the migration class implied by 'name'.
  78     """
  79     try:
  80         module = __import__(app.__name__ + "." + name, '', '', ['Migration'])
  81         return module.Migration
  82     except ImportError:
  83         print " ! Migration %s:%s probably doesn't exist." % (get_app_name(app), name)
  84         print " - Traceback:"
  85         raise
  86
  87
  88 def all_migrations():
  89     return dict([
  90         (app, dict([(name, get_migration(app, name)) for name in get_migration_names(app)]))
  91         for app in get_migrated_apps()
  92     ])
  93
  94
  95 def dependency_tree():
  96     tree = all_migrations()
  97
  98     # Annotate tree with 'backwards edges'
  99     for app, classes in tree.items():
 100         for name, cls in classes.items():
 101             cls.needs = []
 102             if not hasattr(cls, "needed_by"):
 103                 cls.needed_by = []
 104             if hasattr(cls, "depends_on"):
 105                 for dapp, dname in cls.depends_on:
 106                     dapp = get_app(dapp)
 107                     if dapp not in tree:
 108                         print "Migration %s in app %s depends on unmigrated app %s." % (
 109                             name,
 110                             get_app_name(app),
 111                             dapp,
 112                         )
 113                         sys.exit(1)
 114                     if dname not in tree[dapp]:
 115                         print "Migration %s in app %s depends on nonexistent migration %s in app %s." % (
 116                             name,
 117                             get_app_name(app),
 118                             dname,
 119                             get_app_name(dapp),
 120                         )
 121                         sys.exit(1)
 122                     cls.needs.append((dapp, dname))
 123                     if not hasattr(tree[dapp][dname], "needed_by"):
 124                         tree[dapp][dname].needed_by = []
 125                     tree[dapp][dname].needed_by.append((app, name))
 126
 127     # Sanity check whole tree
 128     for app, classes in tree.items():
 129         for name, cls in classes.items():
 130             cls.dependencies = dependencies(tree, app, name)
 131
 132     return tree
 133
 134
 135 def nice_trace(trace):
 136     return " -> ".join([str((get_app_name(a), n)) for a, n in trace])
 137
 138
 139 def dependencies(tree, app, name, trace=[]):
 140     # Copy trace to stop pass-by-ref problems
 141     trace = trace[:]
 142     # Sanity check
 143     for papp, pname in trace:
 144         if app == papp:
 145             if pname == name:
 146                 print "Found circular dependency: %s" % nice_trace(trace + [(app,name)])
 147                 sys.exit(1)
 148             else:
 149                 # See if they depend in the same app the wrong way
 150                 migrations = get_migration_names(app)
 151                 if migrations.index(name) > migrations.index(pname):
 152                     print "Found a lower migration (%s) depending on a higher migration (%s) in the same app (%s)." % (pname, name, get_app_name(app))
 153                     print "Path: %s" % nice_trace(trace + [(app,name)])
 154                     sys.exit(1)
 155     # Get the dependencies of a migration
 156     deps = []
 157     migration = tree[app][name]
 158     for dapp, dname in migration.needs:
 159         deps.extend(
 160             dependencies(tree, dapp, dname, trace+[(app,name)])
 161         )
 162     return deps
 163
 164
 165 def remove_duplicates(l):
 166     m = []
 167     for x in l:
 168         if x not in m:
 169             m.append(x)
 170     return m
 171
 172
 173 def needed_before_forwards(tree, app, name, sameapp=True):
 174     """
 175     Returns a list of migrations that must be applied before (app, name),
 176     in the order they should be applied.
 177     Used to make sure a migration can be applied (and to help apply up to it).
 178     """
 179     app_migrations = get_migration_names(app)
 180     needed = []
 181     if sameapp:
 182         for aname in app_migrations[:app_migrations.index(name)]:
 183             needed += needed_before_forwards(tree, app, aname, False)
 184             needed += [(app, aname)]
 185     for dapp, dname in tree[app][name].needs:
 186         needed += needed_before_forwards(tree, dapp, dname)
 187         needed += [(dapp, dname)]
 188     return remove_duplicates(needed)
 189
 190
 191 def needed_before_backwards(tree, app, name, sameapp=True):
 192     """
 193     Returns a list of migrations that must be unapplied before (app, name) is,
 194     in the order they should be unapplied.
 195     Used to make sure a migration can be unapplied (and to help unapply up to it).
 196     """
 197     app_migrations = get_migration_names(app)
 198     needed = []
 199     if sameapp:
 200         for aname in reversed(app_migrations[app_migrations.index(name)+1:]):
 201             needed += needed_before_backwards(tree, app, aname, False)
 202             needed += [(app, aname)]
 203     for dapp, dname in tree[app][name].needed_by:
 204         needed += needed_before_backwards(tree, dapp, dname)
 205         needed += [(dapp, dname)]
 206     return remove_duplicates(needed)
 207
 208
 209 def run_migrations(toprint, torun, recorder, app, migrations, fake=False, db_dry_run=False, silent=False):
 210     """
 211     Runs the specified migrations forwards, in order.
 212     """
 213     for migration in migrations:
 214         app_name = get_app_name(app)
 215         if not silent:
 216             print toprint % (app_name, migration)
 217         klass = get_migration(app, migration)
 218
 219         if fake:
 220             if not silent:
 221                 print "   (faked)"
 222         else:
 223
 224             # If the database doesn't support running DDL inside a transaction
 225             # *cough*MySQL*cough* then do a dry run first.
 226             if not db.has_ddl_transactions:
 227                 db.dry_run = True
 228                 db.debug, old_debug = False, db.debug
 229                 try:
 230                     getattr(klass(), torun)()
 231                 except:
 232                     traceback.print_exc()
 233                     print " ! Error found during dry run of migration! Aborting."
 234                     return False
 235                 db.debug = old_debug
 236                 db.clear_run_data()
 237
 238             db.dry_run = bool(db_dry_run)
 239
 240             if db.has_ddl_transactions:
 241                 db.start_transaction()
 242             try:
 243                 getattr(klass(), torun)()
 244                 db.execute_deferred_sql()
 245             except:
 246                 if db.has_ddl_transactions:
 247                     db.rollback_transaction()
 248                     raise
 249                 else:
 250                     traceback.print_exc()
 251                     print " ! Error found during real run of migration! Aborting."
 252                     print
 253                     print " ! Since you have a database that does not support running"
 254                     print " ! schema-altering statements in transactions, we have had to"
 255                     print " ! leave it in an interim state between migrations."
 256                     if torun == "forwards":
 257                         print
 258                         print " ! You *might* be able to recover with:"
 259                         db.debug = db.dry_run = True
 260                         klass().backwards()
 261                     print
 262                     print " ! The South developers regret this has happened, and would"
 263                     print " ! like to gently persuade you to consider a slightly"
 264                     print " ! easier-to-deal-with DBMS."
 265                     return False
 266             else:
 267                 if db.has_ddl_transactions:
 268                     db.commit_transaction()
 269
 270         if not db_dry_run:
 271             # Record us as having done this
 272             recorder(app_name, migration)
 273
 274
 275 def run_forwards(app, migrations, fake=False, db_dry_run=False, silent=False):
 276     """
 277     Runs the specified migrations forwards, in order.
 278     """
 279
 280     def record(app_name, migration):
 281         # Record us as having done this
 282         record = MigrationHistory.for_migration(app_name, migration)
 283         record.applied = datetime.datetime.utcnow()
 284         record.save()
 285
 286     return run_migrations(
 287         toprint = " > %s: %s",
 288         torun = "forwards",
 289         recorder = record,
 290         app = app,
 291         migrations = migrations,
 292         fake = fake,
 293         db_dry_run = db_dry_run,
 294         silent = silent,
 295     )
 296
 297
 298 def run_backwards(app, migrations, ignore=[], fake=False, db_dry_run=False, silent=False):
 299     """
 300     Runs the specified migrations backwards, in order, skipping those
 301     migrations in 'ignore'.
 302     """
 303
 304     def record(app_name, migration):
 305         # Record us as having not done this
 306         record = MigrationHistory.for_migration(app_name, migration)
 307         record.delete()
 308
 309     return run_migrations(
 310         toprint = " < %s: %s",
 311         torun = "backwards",
 312         recorder = record,
 313         app = app,
 314         migrations = [x for x in migrations if x not in ignore],
 315         fake = fake,
 316         db_dry_run = db_dry_run,
 317         silent = silent,
 318     )
 319
 320
 321 def right_side_of(x, y):
 322     return left_side_of(reversed(x), reversed(y))
 323
 324
 325 def left_side_of(x, y):
 326     return list(y)[:len(x)] == list(x)
 327
 328
 329 def forwards_problems(tree, forwards, done, silent=False):
 330     problems = []
 331     for app, name in forwards:
 332         if (app, name) not in done:
 333             for dapp, dname in needed_before_backwards(tree, app, name):
 334                 if (dapp, dname) in done:
 335                     if not silent:
 336                         print " ! Migration (%s, %s) should not have been applied before (%s, %s) but was." % (get_app_name(dapp), dname, get_app_name(app), name)
 337                     problems.append(((app, name), (dapp, dname)))
 338     return problems
 339
 340
 341
 342 def backwards_problems(tree, backwards, done, silent=False):
 343     problems = []
 344     for app, name in backwards:
 345         if (app, name) in done:
 346             for dapp, dname in needed_before_forwards(tree, app, name):
 347                 if (dapp, dname) not in done:
 348                     if not silent:
 349                         print " ! Migration (%s, %s) should have been applied before (%s, %s) but wasn't." % (get_app_name(dapp), dname, get_app_name(app), name)
 350                     problems.append(((app, name), (dapp, dname)))
 351     return problems
 352
 353
 354 def migrate_app(app, target_name=None, resolve_mode=None, fake=False, db_dry_run=False, yes=False, silent=False, load_inital_data=False):
 355
 356     app_name = get_app_name(app)
 357
 358     db.debug = not silent
 359
 360     # If any of their app names in the DB contain a ., they're 0.2 or below, so migrate em
 361     longuns = MigrationHistory.objects.filter(app_name__contains=".")
 362     if longuns:
 363         for mh in longuns:
 364             mh.app_name = short_from_long(mh.app_name)
 365             mh.save()
 366         if not silent:
 367             print "- Updated your South 0.2 database."
 368
 369     # Find out what delightful migrations we have
 370     tree = dependency_tree()
 371     migrations = get_migration_names(app)
 372
 373     # If there aren't any, quit quizically
 374     if not migrations:
 375         if not silent:
 376             print "? You have no migrations for the '%s' app. You might want some." % app_name
 377         return
 378
 379     if target_name not in migrations and target_name not in ["zero", None]:
 380         matches = [x for x in migrations if x.startswith(target_name)]
 381         if len(matches) == 1:
 382             target = migrations.index(matches[0]) + 1
 383             if not silent:
 384                 print " - Soft matched migration %s to %s." % (
 385                     target_name,
 386                     matches[0]
 387                 )
 388             target_name = matches[0]
 389         elif len(matches) > 1:
 390             if not silent:
 391                 print " - Prefix %s matches more than one migration:" % target_name
 392                 print "     " + "\n     ".join(matches)
 393             return
 394         else:
 395             if not silent:
 396                 print " ! '%s' is not a migration." % target_name
 397             return
 398
 399     # Check there's no strange ones in the database
 400     ghost_migrations = []
 401     for m in MigrationHistory.objects.filter(applied__isnull = False):
 402         try:
 403             if get_app(m.app_name) not in tree or m.migration not in tree[get_app(m.app_name)]:
 404                 ghost_migrations.append(m)
 405         except ImproperlyConfigured:
 406             pass
 407
 408
 409     if ghost_migrations:
 410         if not silent:
 411             print " ! These migrations are in the database but not on disk:"
 412             print "   - " + "\n   - ".join(["%s: %s" % (x.app_name, x.migration) for x in ghost_migrations])
 413             print " ! I'm not trusting myself; fix this yourself by fiddling"
 414             print " ! with the south_migrationhistory table."
 415         return
 416
 417     # Say what we're doing
 418     if not silent:
 419         print "Running migrations for %s:" % app_name
 420
 421     # Get the forwards and reverse dependencies for this target
 422     if target_name == None:
 423         target_name = migrations[-1]
 424     if target_name == "zero":
 425         forwards = []
 426         backwards = needed_before_backwards(tree, app, migrations[0]) + [(app, migrations[0])]
 427     else:
 428         forwards = needed_before_forwards(tree, app, target_name) + [(app, target_name)]
 429         # When migrating backwards we want to remove up to and including
 430         # the next migration up in this app (not the next one, that includes other apps)
 431         try:
 432             migration_before_here = migrations[migrations.index(target_name)+1]
 433             backwards = needed_before_backwards(tree, app, migration_before_here) + [(app, migration_before_here)]
 434         except IndexError:
 435             backwards = []
 436
 437     # Get the list of currently applied migrations from the db
 438     current_migrations = []
 439     for m in MigrationHistory.objects.filter(applied__isnull = False):
 440         try:
 441             current_migrations.append((get_app(m.app_name), m.migration))
 442         except ImproperlyConfigured:
 443             pass
 444
 445     direction = None
 446     bad = False
 447
 448     # Work out the direction
 449     applied_for_this_app = list(MigrationHistory.objects.filter(app_name=app_name, applied__isnull=False).order_by("migration"))
 450     if target_name == "zero":
 451         direction = -1
 452     elif not applied_for_this_app:
 453         direction = 1
 454     elif migrations.index(target_name) > migrations.index(applied_for_this_app[-1].migration):
 455         direction = 1
 456     elif migrations.index(target_name) < migrations.index(applied_for_this_app[-1].migration):
 457         direction = -1
 458     else:
 459         direction = None
 460
 461     # Is the whole forward branch applied?
 462     missing = [step for step in forwards if step not in current_migrations]
 463     # If they're all applied, we only know it's not backwards
 464     if not missing:
 465         direction = None
 466     # If the remaining migrations are strictly a right segment of the forwards
 467     # trace, we just need to go forwards to our target (and check for badness)
 468     else:
 469         problems = forwards_problems(tree, forwards, current_migrations, silent=silent)
 470         if problems:
 471             bad = True
 472         direction = 1
 473
 474     # What about the whole backward trace then?
 475     if not bad:
 476         missing = [step for step in backwards if step not in current_migrations]
 477         # If they're all missing, stick with the forwards decision
 478         if missing == backwards:
 479             pass
 480         # If what's missing is a strict left segment of backwards (i.e.
 481         # all the higher migrations) then we need to go backwards
 482         else:
 483             problems = backwards_problems(tree, backwards, current_migrations, silent=silent)
 484             if problems:
 485                 bad = True
 486             direction = -1
 487
 488     if bad and resolve_mode not in ['merge']:
 489         if not silent:
 490             print " ! Inconsistent migration history"
 491             print " ! The following options are available:"
 492             print "    --merge: will just attempt the migration ignoring any potential dependency conflicts."
 493         sys.exit(1)
 494
 495     if direction == 1:
 496         if not silent:
 497             print " - Migrating forwards to %s." % target_name
 498         try:
 499             for mapp, mname in forwards:
 500                 if (mapp, mname) not in current_migrations:
 501                     result = run_forwards(mapp, [mname], fake=fake, db_dry_run=db_dry_run, silent=silent)
 502                     if result is False: # The migrations errored, but nicely.
 503                         return
 504         finally:
 505             # Call any pending post_syncdb signals
 506             db.send_pending_create_signals()
 507         # Now load initial data, only if we're really doing things and ended up at current
 508         if not fake and not db_dry_run and load_inital_data and target_name == migrations[-1]:
 509             print " - Loading initial data for %s." % app_name
 510             # Override Django's get_apps call temporarily to only load from the
 511             # current app
 512             old_get_apps, models.get_apps = (
 513                 models.get_apps,
 514                 lambda: [models.get_app(get_app_name(app))],
 515             )
 516             # Load the initial fixture
 517             call_command('loaddata', 'initial_data', verbosity=1)
 518             # Un-override
 519             models.get_apps = old_get_apps
 520     elif direction == -1:
 521         if not silent:
 522             print " - Migrating backwards to just after %s." % target_name
 523         for mapp, mname in backwards:
 524             if (mapp, mname) in current_migrations:
 525                 run_backwards(mapp, [mname], fake=fake, db_dry_run=db_dry_run, silent=silent)
 526     else:
 527         if not silent:
 528             print "- Nothing to migrate."