From 21f878e8112cf1f9b732a6dbb77e70efa68a01aa Mon Sep 17 00:00:00 2001 From: =?utf8?q?Marek=20St=C4=99pniowski?= Date: Wed, 1 Oct 2008 14:50:00 +0200 Subject: [PATCH 1/1] Added south app. --- README | 7 + apps/south/__init__.py | 6 + apps/south/db/__init__.py | 12 + apps/south/db/generic.py | 418 +++++++++++++++++ apps/south/db/mysql.py | 50 ++ apps/south/db/postgresql_psycopg2.py | 32 ++ apps/south/db/sqlite3.py | 12 + apps/south/docs/CHANGELOG | 38 ++ apps/south/docs/CONTRIBUTING | 8 + apps/south/docs/LICENSE | 55 +++ apps/south/docs/README | 10 + apps/south/install/README | 1 + apps/south/install/setup.py | 13 + apps/south/management/__init__.py | 0 apps/south/management/commands/__init__.py | 0 apps/south/management/commands/migrate.py | 55 +++ .../management/commands/startmigration.py | 349 ++++++++++++++ apps/south/management/commands/syncdb.py | 61 +++ apps/south/migration.py | 430 ++++++++++++++++++ apps/south/models.py | 19 + apps/south/tests/__init__.py | 11 + apps/south/tests/db.py | 83 ++++ apps/south/tests/fakeapp/__init__.py | 0 .../tests/fakeapp/migrations/0001_spam.py | 19 + .../tests/fakeapp/migrations/0002_eggs.py | 20 + .../tests/fakeapp/migrations/__init__.py | 0 apps/south/tests/fakeapp/models.py | 0 apps/south/tests/logic.py | 253 +++++++++++ wolnelektury/settings.py | 1 + 29 files changed, 1963 insertions(+) create mode 100644 apps/south/__init__.py create mode 100644 apps/south/db/__init__.py create mode 100644 apps/south/db/generic.py create mode 100644 apps/south/db/mysql.py create mode 100644 apps/south/db/postgresql_psycopg2.py create mode 100644 apps/south/db/sqlite3.py create mode 100644 apps/south/docs/CHANGELOG create mode 100644 apps/south/docs/CONTRIBUTING create mode 100644 apps/south/docs/LICENSE create mode 100644 apps/south/docs/README create mode 100644 apps/south/install/README create mode 100755 apps/south/install/setup.py create mode 100644 apps/south/management/__init__.py create mode 100644 apps/south/management/commands/__init__.py create mode 100644 apps/south/management/commands/migrate.py create mode 100644 apps/south/management/commands/startmigration.py create mode 100644 apps/south/management/commands/syncdb.py create mode 100644 apps/south/migration.py create mode 100644 apps/south/models.py create mode 100644 apps/south/tests/__init__.py create mode 100644 apps/south/tests/db.py create mode 100644 apps/south/tests/fakeapp/__init__.py create mode 100644 apps/south/tests/fakeapp/migrations/0001_spam.py create mode 100644 apps/south/tests/fakeapp/migrations/0002_eggs.py create mode 100644 apps/south/tests/fakeapp/migrations/__init__.py create mode 100644 apps/south/tests/fakeapp/models.py create mode 100644 apps/south/tests/logic.py diff --git a/README b/README index 6dde0816b..1985a8b86 100644 --- a/README +++ b/README @@ -61,6 +61,13 @@ django-newtagging - Typ: biblioteka (aplikacja django) - Nota: Aplikacja wzorowana na [django-tagging](http://code.google.com/p/django-tagging/), która jest również wydana na licencji [MIT](http://www.opensource.org/licenses/mit-license.php) Około połowa kodu jest dzielona. +south +----- +- Źródła: [aercode.org](http://south.aeracode.org/) +- Autorzy: Andrew Godwin , Andy McCurdy +- Licencja: [Apache License 2.0](http://www.opensource.org/licenses/apache2.0.php) +- Typ: biblioteka (aplikacja django) + feedparser ---------- - Źródła: [Google Code](http://code.google.com/p/feedparser/) diff --git a/apps/south/__init__.py b/apps/south/__init__.py new file mode 100644 index 000000000..e797d8585 --- /dev/null +++ b/apps/south/__init__.py @@ -0,0 +1,6 @@ +""" +South - Useable migrations for Django apps +""" + +__version__ = "0.3" +__authors__ = ["Andrew Godwin ", "Andy McCurdy "] \ No newline at end of file diff --git a/apps/south/db/__init__.py b/apps/south/db/__init__.py new file mode 100644 index 000000000..8e4d7733b --- /dev/null +++ b/apps/south/db/__init__.py @@ -0,0 +1,12 @@ + +# Establish the common DatabaseOperations instance, which we call 'db'. +# This code somewhat lifted from django evolution +from django.conf import settings +import sys +module_name = ['south.db', settings.DATABASE_ENGINE] +try: + module = __import__('.'.join(module_name),{},{},['']) +except ImportError: + sys.stderr.write("There is no South database module for the engine '%s'. Please either choose a supported one, or remove South from INSTALLED_APPS.\n" % settings.DATABASE_ENGINE) + sys.exit(1) +db = module.DatabaseOperations() \ No newline at end of file diff --git a/apps/south/db/generic.py b/apps/south/db/generic.py new file mode 100644 index 000000000..09dde0362 --- /dev/null +++ b/apps/south/db/generic.py @@ -0,0 +1,418 @@ + +from django.core.management.color import no_style +from django.db import connection, transaction, models +from django.db.backends.util import truncate_name +from django.dispatch import dispatcher +from django.conf import settings + +class DatabaseOperations(object): + + """ + Generic SQL implementation of the DatabaseOperations. + Some of this code comes from Django Evolution. + """ + + def __init__(self): + self.debug = False + self.deferred_sql = [] + + + def execute(self, sql, params=[]): + """ + Executes the given SQL statement, with optional parameters. + If the instance's debug attribute is True, prints out what it executes. + """ + cursor = connection.cursor() + if self.debug: + print " = %s" % sql, params + cursor.execute(sql, params) + try: + return cursor.fetchall() + except: + return [] + + + def add_deferred_sql(self, sql): + """ + Add a SQL statement to the deferred list, that won't be executed until + this instance's execute_deferred_sql method is run. + """ + self.deferred_sql.append(sql) + + + def execute_deferred_sql(self): + """ + Executes all deferred SQL, resetting the deferred_sql list + """ + for sql in self.deferred_sql: + self.execute(sql) + + self.deferred_sql = [] + + + def create_table(self, table_name, fields): + """ + Creates the table 'table_name'. 'fields' is a tuple of fields, + each repsented by a 2-part tuple of field name and a + django.db.models.fields.Field object + """ + qn = connection.ops.quote_name + columns = [ + self.column_sql(table_name, field_name, field) + for field_name, field in fields + ] + + self.execute('CREATE TABLE %s (%s);' % (qn(table_name), ', '.join([col for col in columns if col]))) + + add_table = create_table # Alias for consistency's sake + + + def rename_table(self, old_table_name, table_name): + """ + Renames the table 'old_table_name' to 'table_name'. + """ + if old_table_name == table_name: + # No Operation + return + qn = connection.ops.quote_name + params = (qn(old_table_name), qn(table_name)) + self.execute('ALTER TABLE %s RENAME TO %s;' % params) + + + def delete_table(self, table_name): + """ + Deletes the table 'table_name'. + """ + qn = connection.ops.quote_name + params = (qn(table_name), ) + self.execute('DROP TABLE %s;' % params) + + drop_table = delete_table + + + def add_column(self, table_name, name, field): + """ + Adds the column 'name' to the table 'table_name'. + Uses the 'field' paramater, a django.db.models.fields.Field instance, + to generate the necessary sql + + @param table_name: The name of the table to add the column to + @param name: The name of the column to add + @param field: The field to use + """ + qn = connection.ops.quote_name + sql = self.column_sql(table_name, name, field) + if sql: + params = ( + qn(table_name), + sql, + ) + sql = 'ALTER TABLE %s ADD COLUMN %s;' % params + self.execute(sql) + + + alter_string_set_type = 'ALTER COLUMN %(column)s TYPE %(type)s' + alter_string_set_null = 'ALTER COLUMN %(column)s SET NOT NULL' + alter_string_drop_null = 'ALTER COLUMN %(column)s DROP NOT NULL' + + def alter_column(self, table_name, name, field): + """ + Alters the given column name so it will match the given field. + Note that conversion between the two by the database must be possible. + + @param table_name: The name of the table to add the column to + @param name: The name of the column to alter + @param field: The new field definition to use + """ + + # hook for the field to do any resolution prior to it's attributes being queried + if hasattr(field, 'south_init'): + field.south_init() + + qn = connection.ops.quote_name + + # First, change the type + params = { + "column": qn(name), + "type": field.db_type(), + } + sqls = [self.alter_string_set_type % params] + + + # Next, set any default + params = ( + qn(name), + ) + + if not field.null and field.has_default(): + default = field.get_default() + if isinstance(default, basestring): + default = "'%s'" % default + params += ("SET DEFAULT %s",) + else: + params += ("DROP DEFAULT",) + + sqls.append('ALTER COLUMN %s %s ' % params) + + + # Next, nullity + params = { + "column": qn(name), + "type": field.db_type(), + } + if field.null: + sqls.append(self.alter_string_drop_null % params) + else: + sqls.append(self.alter_string_set_null % params) + + + # TODO: Unique + + self.execute("ALTER TABLE %s %s;" % (qn(table_name), ", ".join(sqls))) + + + def column_sql(self, table_name, field_name, field, tablespace=''): + """ + Creates the SQL snippet for a column. Used by add_column and add_table. + """ + qn = connection.ops.quote_name + + field.set_attributes_from_name(field_name) + + # hook for the field to do any resolution prior to it's attributes being queried + if hasattr(field, 'south_init'): + field.south_init() + + sql = field.db_type() + if sql: + field_output = [qn(field.column), sql] + field_output.append('%sNULL' % (not field.null and 'NOT ' or '')) + if field.primary_key: + field_output.append('PRIMARY KEY') + elif field.unique: + field_output.append('UNIQUE') + + tablespace = field.db_tablespace or tablespace + if tablespace and connection.features.supports_tablespaces and field.unique: + # We must specify the index tablespace inline, because we + # won't be generating a CREATE INDEX statement for this field. + field_output.append(connection.ops.tablespace_sql(tablespace, inline=True)) + + sql = ' '.join(field_output) + sqlparams = () + # if the field is "NOT NULL" and a default value is provided, create the column with it + # this allows the addition of a NOT NULL field to a table with existing rows + if not field.null and field.has_default(): + default = field.get_default() + if isinstance(default, basestring): + default = "'%s'" % default.replace("'", "''") + sql += " DEFAULT %s" + sqlparams = (default) + + if field.rel: + self.add_deferred_sql( + self.foreign_key_sql( + table_name, + field.column, + field.rel.to._meta.db_table, + field.rel.to._meta.get_field(field.rel.field_name).column + ) + ) + + if field.db_index and not field.unique: + self.add_deferred_sql(self.create_index_sql(table_name, [field.column])) + + if hasattr(field, 'post_create_sql'): + style = no_style() + for stmt in field.post_create_sql(style, table_name): + self.add_deferred_sql(stmt) + + if sql: + return sql % sqlparams + else: + return None + + def foreign_key_sql(self, from_table_name, from_column_name, to_table_name, to_column_name): + """ + Generates a full SQL statement to add a foreign key constraint + """ + constraint_name = '%s_refs_%s_%x' % (from_column_name, to_column_name, abs(hash((from_table_name, to_table_name)))) + return 'ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' % ( + from_table_name, + truncate_name(constraint_name, connection.ops.max_name_length()), + from_column_name, + to_table_name, + to_column_name, + connection.ops.deferrable_sql() # Django knows this + ) + + def create_index_name(self, table_name, column_names): + """ + Generate a unique name for the index + """ + index_unique_name = '' + if len(column_names) > 1: + index_unique_name = '_%x' % abs(hash((table_name, ','.join(column_names)))) + + return '%s_%s%s' % (table_name, column_names[0], index_unique_name) + + def create_index_sql(self, table_name, column_names, unique=False, db_tablespace=''): + """ + Generates a create index statement on 'table_name' for a list of 'column_names' + """ + if not column_names: + print "No column names supplied on which to create an index" + return '' + + if db_tablespace and connection.features.supports_tablespaces: + tablespace_sql = ' ' + connection.ops.tablespace_sql(db_tablespace) + else: + tablespace_sql = '' + + index_name = self.create_index_name(table_name, column_names) + qn = connection.ops.quote_name + return 'CREATE %sINDEX %s ON %s (%s)%s;' % ( + unique and 'UNIQUE ' or '', + index_name, + table_name, + ','.join([qn(field) for field in column_names]), + tablespace_sql + ) + + def create_index(self, table_name, column_names, unique=False, db_tablespace=''): + """ Executes a create index statement """ + sql = self.create_index_sql(table_name, column_names, unique, db_tablespace) + self.execute(sql) + + + def delete_index(self, table_name, column_names, db_tablespace=''): + """ + Deletes an index created with create_index. + This is possible using only columns due to the deterministic + index naming function which relies on column names. + """ + name = self.create_index_name(table_name, column_names) + sql = "DROP INDEX %s" % name + self.execute(sql) + + + def delete_column(self, table_name, name): + """ + Deletes the column 'column_name' from the table 'table_name'. + """ + qn = connection.ops.quote_name + params = (qn(table_name), qn(name)) + self.execute('ALTER TABLE %s DROP COLUMN %s CASCADE;' % params, []) + + + def rename_column(self, table_name, old, new): + """ + Renames the column 'old' from the table 'table_name' to 'new'. + """ + raise NotImplementedError("rename_column has no generic SQL syntax") + + + def start_transaction(self): + """ + Makes sure the following commands are inside a transaction. + Must be followed by a (commit|rollback)_transaction call. + """ + transaction.commit_unless_managed() + transaction.enter_transaction_management() + transaction.managed(True) + + + def commit_transaction(self): + """ + Commits the current transaction. + Must be preceded by a start_transaction call. + """ + transaction.commit() + transaction.leave_transaction_management() + + + def rollback_transaction(self): + """ + Rolls back the current transaction. + Must be preceded by a start_transaction call. + """ + transaction.rollback() + transaction.leave_transaction_management() + + + def send_create_signal(self, app_label, model_names): + """ + Sends a post_syncdb signal for the model specified. + + If the model is not found (perhaps it's been deleted?), + no signal is sent. + + TODO: The behavior of django.contrib.* apps seems flawed in that + they don't respect created_models. Rather, they blindly execute + over all models within the app sending the signal. This is a + patch we should push Django to make For now, this should work. + """ + app = models.get_app(app_label) + if not app: + return + + created_models = [] + for model_name in model_names: + model = models.get_model(app_label, model_name) + if model: + created_models.append(model) + + if created_models: + # syncdb defaults -- perhaps take these as options? + verbosity = 1 + interactive = True + + if hasattr(dispatcher, "send"): + dispatcher.send(signal=models.signals.post_syncdb, sender=app, + app=app, created_models=created_models, + verbosity=verbosity, interactive=interactive) + else: + models.signals.post_syncdb.send(sender=app, + app=app, created_models=created_models, + verbosity=verbosity, interactive=interactive) + + def mock_model(self, model_name, db_table, db_tablespace='', + pk_field_name='id', pk_field_type=models.AutoField, + pk_field_kwargs={}): + """ + Generates a MockModel class that provides enough information + to be used by a foreign key/many-to-many relationship. + + Migrations should prefer to use these rather than actual models + as models could get deleted over time, but these can remain in + migration files forever. + """ + class MockOptions(object): + def __init__(self): + self.db_table = db_table + self.db_tablespace = db_tablespace or settings.DEFAULT_TABLESPACE + self.object_name = model_name + self.module_name = model_name.lower() + + if pk_field_type == models.AutoField: + pk_field_kwargs['primary_key'] = True + + self.pk = pk_field_type(**pk_field_kwargs) + self.pk.set_attributes_from_name(pk_field_name) + self.abstract = False + + def get_field_by_name(self, field_name): + # we only care about the pk field + return (self.pk, self.model, True, False) + + def get_field(self, name): + # we only care about the pk field + return self.pk + + class MockModel(object): + _meta = None + + # We need to return an actual class object here, not an instance + MockModel._meta = MockOptions() + MockModel._meta.model = MockModel + return MockModel diff --git a/apps/south/db/mysql.py b/apps/south/db/mysql.py new file mode 100644 index 000000000..c3659fc48 --- /dev/null +++ b/apps/south/db/mysql.py @@ -0,0 +1,50 @@ + +from django.db import connection +from south.db import generic + +class DatabaseOperations(generic.DatabaseOperations): + + """ + MySQL implementation of database operations. + """ + + alter_string_set_type = '' + alter_string_set_null = 'MODIFY %(column)s %(type)s NULL;' + alter_string_drop_null = 'MODIFY %(column)s %(type)s NOT NULL;' + + def rename_column(self, table_name, old, new): + if old == new: + return [] + + qn = connection.ops.quote_name + + rows = [x for x in self.execute('DESCRIBE %s' % (qn(table_name),)) if x[0] == old] + + if not rows: + raise ValueError("No column '%s' in '%s'." % (old, table_name)) + + params = ( + qn(table_name), + qn(old), + qn(new), + "%s %s %s %s %s" % ( + rows[0][1], + rows[0][2] == "YES" and "NULL" or "NOT NULL", + rows[0][3] == "PRI" and "PRIMARY KEY" or "", + rows[0][4] and "DEFAULT %s" % rows[0][4] or "", + rows[0][5] or "", + ), + ) + self.execute('ALTER TABLE %s CHANGE COLUMN %s %s %s;' % params) + + + def rename_table(self, old_table_name, table_name): + """ + Renames the table 'old_table_name' to 'table_name'. + """ + if old_table_name == table_name: + # No Operation + return + qn = connection.ops.quote_name + params = (qn(old_table_name), qn(table_name)) + self.execute('RENAME TABLE %s TO %s;' % params) \ No newline at end of file diff --git a/apps/south/db/postgresql_psycopg2.py b/apps/south/db/postgresql_psycopg2.py new file mode 100644 index 000000000..278eb3e9f --- /dev/null +++ b/apps/south/db/postgresql_psycopg2.py @@ -0,0 +1,32 @@ + +from django.db import connection +from south.db import generic + +class DatabaseOperations(generic.DatabaseOperations): + + """ + PsycoPG2 implementation of database operations. + """ + + def rename_column(self, table_name, old, new): + if old == new: + return [] + qn = connection.ops.quote_name + params = (qn(table_name), qn(old), qn(new)) + self.execute('ALTER TABLE %s RENAME COLUMN %s TO %s;' % params) + + def rename_table(self, old_table_name, table_name): + # First, rename the table + generic.DatabaseOperations.rename_table(self, old_table_name, table_name) + # Then, try renaming the ID sequence + # (if you're using other AutoFields... your problem, unfortunately) + self.commit_transaction() + self.start_transaction() + try: + generic.DatabaseOperations.rename_table(self, old_table_name+"_id_seq", table_name+"_id_seq") + except: + print " ~ No such sequence (ignoring error)" + self.rollback_transaction() + else: + self.commit_transaction() + self.start_transaction() \ No newline at end of file diff --git a/apps/south/db/sqlite3.py b/apps/south/db/sqlite3.py new file mode 100644 index 000000000..6073b4d4c --- /dev/null +++ b/apps/south/db/sqlite3.py @@ -0,0 +1,12 @@ + +from django.db import connection +from south.db import generic + +class DatabaseOperations(generic.DatabaseOperations): + + """ + SQLite3 implementation of database operations. + """ + + def __init__(self): + raise NotImplementedError("Support for SQLite3 is not yet complete.") \ No newline at end of file diff --git a/apps/south/docs/CHANGELOG b/apps/south/docs/CHANGELOG new file mode 100644 index 000000000..fa106f976 --- /dev/null +++ b/apps/south/docs/CHANGELOG @@ -0,0 +1,38 @@ +0.3 +=== + +The yay-dependencies release. + +New features: + + - Dependency support for migrations + - Fields are now used for declaring columns and tables + + +0.2 +=== + +The oh-i'm-sorry-mysql-users release. + +New features: + + - MySQL support up to the same level as PostgreSQL + - New --all option to ./manage.py startmigration, which creates a migration + for every model in the given app. For project starts. + - Project status upgraded to 'beta'. Next up, a colour-coded + release level system. + +Fixed bugs: + + - A few typos in various column methods + - ManyToMany tables weren't created by startmigration migrations. + +Known bugs: + + - None + + +0.1 +=== + +Initial release. diff --git a/apps/south/docs/CONTRIBUTING b/apps/south/docs/CONTRIBUTING new file mode 100644 index 000000000..56dd52572 --- /dev/null +++ b/apps/south/docs/CONTRIBUTING @@ -0,0 +1,8 @@ + +Contributing to South +--------------------- + +Contributions to South are very welcome. + + +You can find more info on our site at http://south.aeracode.org/wiki/Contributing diff --git a/apps/south/docs/LICENSE b/apps/south/docs/LICENSE new file mode 100644 index 000000000..1914f856a --- /dev/null +++ b/apps/south/docs/LICENSE @@ -0,0 +1,55 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and + + 2. You must cause any modified files to carry prominent notices stating that You changed the files; and + + 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/apps/south/docs/README b/apps/south/docs/README new file mode 100644 index 000000000..99d3f20de --- /dev/null +++ b/apps/south/docs/README @@ -0,0 +1,10 @@ +This is South, a Django application to provide migrations in a sane way. + +By sane, we mean that the status of every migration is tracked individually, +rather than just the number of the top migration reached; this means South +can detect when you have an unapplied migration that's sitting in the middle +of a whole load of applied ones, and will let you apply it straight off, +or let you roll back to it, and apply from there forward. + +Documentation on South is currently available on our project site; +you can find it at http://south.aeracode.org/wiki/Documentation diff --git a/apps/south/install/README b/apps/south/install/README new file mode 100644 index 000000000..897b51d34 --- /dev/null +++ b/apps/south/install/README @@ -0,0 +1 @@ +To use this setup.py, make sure you checked out this trunk or branch into a directory called 'south', copy the setup.py into the directory above it, and off you go. diff --git a/apps/south/install/setup.py b/apps/south/install/setup.py new file mode 100755 index 000000000..6da3d4afc --- /dev/null +++ b/apps/south/install/setup.py @@ -0,0 +1,13 @@ +#!/usr/bin/python + +from setuptools import setup, find_packages + +setup( + name='South', + version='0.3', + description='South: Migrations for Django', + author='Andrew Godwin & Andy McCurdy', + author_email='south@aeracode.org', + url='http://south.aeracode.org/', + packages=["south", "south.db", "south.management", "south.management.commands", "south.tests"], +) diff --git a/apps/south/management/__init__.py b/apps/south/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/south/management/commands/__init__.py b/apps/south/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/south/management/commands/migrate.py b/apps/south/management/commands/migrate.py new file mode 100644 index 000000000..d2d69982f --- /dev/null +++ b/apps/south/management/commands/migrate.py @@ -0,0 +1,55 @@ +from django.core.management.base import BaseCommand +from django.core.management.color import no_style +from django.conf import settings +from django.db import models +from optparse import make_option +from south import migration +import sys + +class Command(BaseCommand): + option_list = BaseCommand.option_list + ( + make_option('--skip', action='store_true', dest='skip', default=False, + help='Will skip over out-of-order missing migrations'), + make_option('--merge', action='store_true', dest='merge', default=False, + help='Will run out-of-order missing migrations as they are - no rollbacks.'), + make_option('--only', action='store_true', dest='only', default=False, + help='Only runs or rolls back the migration specified, and none around it.'), + make_option('--fake', action='store_true', dest='fake', default=False, + help="Pretends to do the migrations, but doesn't actually execute them."), + ) + help = "Runs migrations for all apps." + + def handle(self, app=None, target=None, skip=False, merge=False, only=False, backwards=False, fake=False, **options): + + # Work out what the resolve mode is + resolve_mode = merge and "merge" or (skip and "skip" or None) + # Turn on db debugging + from south.db import db + db.debug = True + + # NOTE: THIS IS DUPLICATED FROM django.core.management.commands.syncdb + # This code imports any module named 'management' in INSTALLED_APPS. + # The 'management' module is the preferred way of listening to post_syncdb + # signals, and since we're sending those out with create_table migrations, + # we need apps to behave correctly. + for app_name in settings.INSTALLED_APPS: + try: + __import__(app_name + '.management', {}, {}, ['']) + except ImportError, exc: + msg = exc.args[0] + if not msg.startswith('No module named') or 'management' not in msg: + raise + # END DJANGO DUPE CODE + + # Migrate each app + if app: + apps = [migration.get_app(app)] + else: + apps = migration.get_migrated_apps() + for app in apps: + migration.migrate_app( + app, + resolve_mode = resolve_mode, + target_name = target, + fake = fake, + ) diff --git a/apps/south/management/commands/startmigration.py b/apps/south/management/commands/startmigration.py new file mode 100644 index 000000000..f52efe7aa --- /dev/null +++ b/apps/south/management/commands/startmigration.py @@ -0,0 +1,349 @@ +from django.core.management.base import BaseCommand +from django.core.management.color import no_style +from django.db import models +from django.db.models.fields.related import RECURSIVE_RELATIONSHIP_CONSTANT +from django.contrib.contenttypes.generic import GenericRelation +from optparse import make_option +from south import migration +import sys +import os +import re +import string +import random +import inspect +import parser + +class Command(BaseCommand): + option_list = BaseCommand.option_list + ( + make_option('--model', action='append', dest='model_list', type='string', + help='Generate a Create Table migration for the specified model. Add multiple models to this migration with subsequent --model parameters.'), + make_option('--initial', action='store_true', dest='initial', default=False, + help='Generate the initial schema for the app.'), + ) + help = "Creates a new template migration for the given app" + + def handle(self, app=None, name="", model_list=None, initial=False, **options): + + # If model_list is None, then it's an empty list + model_list = model_list or [] + + # make sure --model and --all aren't both specified + if initial and model_list: + print "You cannot use --initial and other options together" + return + + # specify the default name 'initial' if a name wasn't specified and we're + # doing a migration for an entire app + if not name and initial: + name = 'initial' + + # if not name, there's an error + if not name: + print "You must name this migration" + return + + if not app: + print "Please provide an app in which to create the migration." + return + + # See if the app exists + app_models_module = models.get_app(app) + if not app_models_module: + print "App '%s' doesn't seem to exist, isn't in INSTALLED_APPS, or has no models." % app + return + + # Determine what models should be included in this migration. + models_to_migrate = [] + if initial: + models_to_migrate = models.get_models(app_models_module) + if not models_to_migrate: + print "No models found in app '%s'" % (app) + return + else: + for model_name in model_list: + model = models.get_model(app, model_name) + if not model: + print "Couldn't find model '%s' in app '%s'" % (model_name, app) + return + + models_to_migrate.append(model) + + # Make the migrations directory if it's not there + app_module_path = app_models_module.__name__.split('.')[0:-1] + try: + app_module = __import__('.'.join(app_module_path), {}, {}, ['']) + except ImportError: + print "Couldn't find path to App '%s'." % app + return + + migrations_dir = os.path.join( + os.path.dirname(app_module.__file__), + "migrations", + ) + if not os.path.isdir(migrations_dir): + print "Creating migrations directory at '%s'..." % migrations_dir + os.mkdir(migrations_dir) + # Touch the init py file + open(os.path.join(migrations_dir, "__init__.py"), "w").close() + # See what filename is next in line. We assume they use numbers. + migrations = migration.get_migration_names(migration.get_app(app)) + highest_number = 0 + for migration_name in migrations: + try: + number = int(migration_name.split("_")[0]) + highest_number = max(highest_number, number) + except ValueError: + pass + # Make the new filename + new_filename = "%04i%s_%s.py" % ( + highest_number + 1, + "".join([random.choice(string.letters.lower()) for i in range(0)]), # Possible random stuff insertion + name, + ) + # If there's a model, make the migration skeleton, else leave it bare + forwards, backwards = '', '' + if models_to_migrate: + for model in models_to_migrate: + table_name = model._meta.db_table + mock_models = [] + fields = [] + for f in model._meta.local_fields: + # look up the field definition to see how this was created + field_definition = generate_field_definition(model, f) + if field_definition: + + if isinstance(f, models.ForeignKey): + mock_models.append(create_mock_model(f.rel.to)) + field_definition = related_field_definition(f, field_definition) + + else: + print "Warning: Could not generate field definition for %s.%s, manual editing of migration required." % \ + (model._meta.object_name, f.name) + + field_definition = '<<< REPLACE THIS WITH FIELD DEFINITION FOR %s.%s >>>' % (model._meta.object_name, f.name) + + fields.append((f.name, field_definition)) + + if mock_models: + forwards += ''' + + # Mock Models + %s + ''' % "\n ".join(mock_models) + + forwards += ''' + # Model '%s' + db.create_table('%s', ( + %s + ))''' % ( + model._meta.object_name, + table_name, + "\n ".join(["('%s', %s)," % (f[0], f[1]) for f in fields]), + ) + + backwards = ('''db.delete_table('%s') + ''' % table_name) + backwards + + # Now go through local M2Ms and add extra stuff for them + for m in model._meta.local_many_to_many: + # ignore generic relations + if isinstance(m, GenericRelation): + continue + + # if the 'through' option is specified, the table will + # be created through the normal model creation above. + if m.rel.through: + continue + + mock_models = [create_mock_model(model), create_mock_model(m.rel.to)] + + forwards += ''' + # Mock Models + %s + + # M2M field '%s.%s' + db.create_table('%s', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('%s', models.ForeignKey(%s, null=False)), + ('%s', models.ForeignKey(%s, null=False)) + )) ''' % ( + "\n ".join(mock_models), + model._meta.object_name, + m.name, + m.m2m_db_table(), + m.m2m_column_name()[:-3], # strip off the '_id' at the end + model._meta.object_name, + m.m2m_reverse_name()[:-3], # strip off the '_id' at the ned + m.rel.to._meta.object_name + ) + + backwards = '''db.delete_table('%s') + ''' % m.m2m_db_table() + backwards + + if model._meta.unique_together: + ut = model._meta.unique_together + if not isinstance(ut[0], (list, tuple)): + ut = (ut,) + + for unique in ut: + columns = ["'%s'" % model._meta.get_field(f).column for f in unique] + + forwards += ''' + db.create_index('%s', [%s], unique=True, db_tablespace='%s') + ''' % ( + table_name, + ','.join(columns), + model._meta.db_tablespace + ) + + + forwards += ''' + + db.send_create_signal('%s', ['%s'])''' % ( + app, + "','".join(model._meta.object_name for model in models_to_migrate) + ) + + else: + forwards = '"Write your forwards migration here"' + backwards = '"Write your backwards migration here"' + fp = open(os.path.join(migrations_dir, new_filename), "w") + fp.write(""" +from south.db import db +from %s.models import * + +class Migration: + + def forwards(self): + %s + + def backwards(self): + %s +""" % ('.'.join(app_module_path), forwards, backwards)) + fp.close() + print "Created %s." % new_filename + + +def generate_field_definition(model, field): + """ + Inspects the source code of 'model' to find the code used to generate 'field' + """ + def test_field(field_definition): + try: + parser.suite(field_definition) + return True + except SyntaxError: + return False + + def strip_comments(field_definition): + # remove any comments at the end of the field definition string. + field_definition = field_definition.strip() + if '#' not in field_definition: + return field_definition + + index = field_definition.index('#') + while index: + stripped_definition = field_definition[:index].strip() + # if the stripped definition is parsable, then we've removed + # the correct comment. + if test_field(stripped_definition): + return stripped_definition + + index = field_definition.index('#', index+1) + + return field_definition + + # give field subclasses a chance to do anything tricky + # with the field definition + if hasattr(field, 'south_field_definition'): + return field.south_field_definition() + + field_pieces = [] + found_field = False + source = inspect.getsourcelines(model) + if not source: + raise Exception("Could not find source to model: '%s'" % (model.__name__)) + + # look for a line starting with the field name + start_field_re = re.compile(r'\s*%s\s*=\s*(.*)' % field.name) + for line in source[0]: + # if the field was found during a previous iteration, + # we're here because the field spans across multiple lines + # append the current line and try again + if found_field: + field_pieces.append(line.strip()) + if test_field(' '.join(field_pieces)): + return strip_comments(' '.join(field_pieces)) + continue + + match = start_field_re.match(line) + if match: + found_field = True + field_pieces.append(match.groups()[0].strip()) + if test_field(' '.join(field_pieces)): + return strip_comments(' '.join(field_pieces)) + + # the 'id' field never gets defined, so return what django does by default + # django.db.models.options::_prepare + if field.name == 'id' and field.__class__ == models.AutoField: + return "models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)" + + # search this classes parents + for base in model.__bases__: + # we don't want to scan the django base model + if base == models.Model: + continue + + field_definition = generate_field_definition(base, field) + if field_definition: + return field_definition + + return None + +def replace_model_string(field_definition, search_string, model_name): + # wrap 'search_string' in both ' and " chars when searching + quotes = ["'", '"'] + for quote in quotes: + test = "%s%s%s" % (quote, search_string, quote) + if test in field_definition: + return field_definition.replace(test, model_name) + + return None + +def related_field_definition(field, field_definition): + # if the field definition contains any of the following strings, + # replace them with the model definition: + # applabel.modelname + # modelname + # django.db.models.fields.related.RECURSIVE_RELATIONSHIP_CONSTANT + strings = [ + '%s.%s' % (field.rel.to._meta.app_label, field.rel.to._meta.object_name), + '%s' % field.rel.to._meta.object_name, + RECURSIVE_RELATIONSHIP_CONSTANT + ] + + for test in strings: + fd = replace_model_string(field_definition, test, field.rel.to._meta.object_name) + if fd: + return fd + + return field_definition + +def create_mock_model(model): + # produce a string representing the python syntax necessary for creating + # a mock model using the supplied real model + if model._meta.pk.__class__.__module__ != 'django.db.models.fields': + # we can fix this with some clever imports, but it doesn't seem necessary to + # spend time on just yet + print "Can't generate a mock model for %s because it's primary key isn't a default django field" % model + sys.exit() + + return "%s = db.mock_model(model_name='%s', db_table='%s', db_tablespace='%s', pk_field_name='%s', pk_field_type=models.%s)" % \ + ( + model._meta.object_name, + model._meta.object_name, + model._meta.db_table, + model._meta.db_tablespace, + model._meta.pk.name, + model._meta.pk.__class__.__name__ + ) \ No newline at end of file diff --git a/apps/south/management/commands/syncdb.py b/apps/south/management/commands/syncdb.py new file mode 100644 index 000000000..6ffc12015 --- /dev/null +++ b/apps/south/management/commands/syncdb.py @@ -0,0 +1,61 @@ +from django.core.management.base import NoArgsCommand +from django.core.management.color import no_style +from django.utils.datastructures import SortedDict +from optparse import make_option +from south import migration +from django.core.management.commands import syncdb +from django.conf import settings +from django.db import models +from django.db.models.loading import cache +from django.core import management +import sys + +def get_app_name(app): + return '.'.join( app.__name__.split('.')[0:-1] ) + +class Command(NoArgsCommand): + option_list = NoArgsCommand.option_list + ( + make_option('--verbosity', action='store', dest='verbosity', default='1', + type='choice', choices=['0', '1', '2'], + help='Verbosity level; 0=minimal output, 1=normal output, 2=all output'), + make_option('--noinput', action='store_false', dest='interactive', default=True, + help='Tells Django to NOT prompt the user for input of any kind.'), + make_option('--migrate', action='store_true', dest='migrate', default=False, + help='Tells South to also perform migrations after the sync. Default for during testing, and other internal calls.'), + ) + help = "Create the database tables for all apps in INSTALLED_APPS whose tables haven't already been created, except those which use migrations." + + def handle_noargs(self, **options): + # Work out what uses migrations and so doesn't need syncing + apps_needing_sync = [] + apps_migrated = [] + for app in models.get_apps(): + app_name = get_app_name(app) + migrations = migration.get_app(app) + if migrations is None: + apps_needing_sync.append(app_name) + else: + # This is a migrated app, leave it + apps_migrated.append(app_name) + # Run syncdb on only the ones needed + print "Syncing..." + old_installed, settings.INSTALLED_APPS = settings.INSTALLED_APPS, apps_needing_sync + old_app_store, cache.app_store = cache.app_store, SortedDict([ + (k, v) for (k, v) in cache.app_store.items() + if get_app_name(k) in apps_needing_sync + ]) + syncdb.Command().execute(**options) + settings.INSTALLED_APPS = old_installed + cache.app_store = old_app_store + # Migrate if needed + if options.get('migrate', True): + print "Migrating..." + management.call_command('migrate') + # Be obvious about what we did + print "\nSynced:\n > %s" % "\n > ".join(apps_needing_sync) + + if options.get('migrate', True): + print "\nMigrated:\n - %s" % "\n - ".join(apps_migrated) + else: + print "\nNot synced (use migrations):\n - %s" % "\n - ".join(apps_migrated) + print "(use ./manage.py migrate to migrate these)" diff --git a/apps/south/migration.py b/apps/south/migration.py new file mode 100644 index 000000000..c650c5475 --- /dev/null +++ b/apps/south/migration.py @@ -0,0 +1,430 @@ + +import datetime +import os +import sys +from django.conf import settings +from django.db import models +from models import MigrationHistory +from south.db import db + + +def get_app(app): + """ + Returns the migrations module for the given app model name/module, or None + if it does not use migrations. + """ + if isinstance(app, (str, unicode)): + # If it's a string, use the models module + app = models.get_app(app) + mod = __import__(app.__name__[:-7], {}, {}, ['migrations']) + if hasattr(mod, 'migrations'): + return getattr(mod, 'migrations') + + +def get_migrated_apps(): + """ + Returns all apps with migrations. + """ + for mapp in models.get_apps(): + app = get_app(mapp) + if app: + yield app + + +def get_app_name(app): + """ + Returns the _internal_ app name for the given app module. + i.e. for will return 'auth' + """ + return app.__name__.split('.')[-2] + + +def get_app_fullname(app): + """ + Returns the full python name of an app - e.g. django.contrib.auth + """ + return app.__name__[:-11] + + +def short_from_long(app_name): + return app_name.split(".")[-1] + + +def get_migration_names(app): + """ + Returns a list of migration file names for the given app. + """ + return sorted([ + filename[:-3] + for filename in os.listdir(os.path.dirname(app.__file__)) + if filename.endswith(".py") and filename != "__init__.py" + ]) + + +def get_migration_classes(app): + """ + Returns a list of migration classes (one for each migration) for the app. + """ + for name in get_migration_names(app): + yield get_migration(app, name) + + +def get_migration(app, name): + """ + Returns the migration class implied by 'name'. + """ + try: + module = __import__(app.__name__ + "." + name, '', '', ['Migration']) + return module.Migration + except ImportError: + raise ValueError("Migration %s:%s does not exist." % (get_app_name(app), name)) + + +def all_migrations(): + return dict([ + (app, dict([(name, get_migration(app, name)) for name in get_migration_names(app)])) + for app in get_migrated_apps() + ]) + + +def dependency_tree(): + tree = all_migrations() + + # Annotate tree with 'backwards edges' + for app, classes in tree.items(): + for name, cls in classes.items(): + cls.needs = [] + if not hasattr(cls, "needed_by"): + cls.needed_by = [] + if hasattr(cls, "depends_on"): + for dapp, dname in cls.depends_on: + dapp = get_app(dapp) + if dapp not in tree: + print "Migration %s in app %s depends on unmigrated app %s." % ( + name, + get_app_name(app), + dapp, + ) + sys.exit(1) + if dname not in tree[dapp]: + print "Migration %s in app %s depends on nonexistent migration %s in app %s." % ( + name, + get_app_name(app), + dname, + get_app_name(dapp), + ) + sys.exit(1) + cls.needs.append((dapp, dname)) + if not hasattr(tree[dapp][dname], "needed_by"): + tree[dapp][dname].needed_by = [] + tree[dapp][dname].needed_by.append((app, name)) + + # Sanity check whole tree + for app, classes in tree.items(): + for name, cls in classes.items(): + cls.dependencies = dependencies(tree, app, name) + + return tree + + +def nice_trace(trace): + return " -> ".join([str((get_app_name(a), n)) for a, n in trace]) + + +def dependencies(tree, app, name, trace=[]): + # Copy trace to stop pass-by-ref problems + trace = trace[:] + # Sanity check + for papp, pname in trace: + if app == papp: + if pname == name: + print "Found circular dependency: %s" % nice_trace(trace + [(app,name)]) + sys.exit(1) + else: + # See if they depend in the same app the wrong way + migrations = get_migration_names(app) + if migrations.index(name) > migrations.index(pname): + print "Found a lower migration (%s) depending on a higher migration (%s) in the same app (%s)." % (pname, name, get_app_name(app)) + print "Path: %s" % nice_trace(trace + [(app,name)]) + sys.exit(1) + # Get the dependencies of a migration + deps = [] + migration = tree[app][name] + for dapp, dname in migration.needs: + deps.extend( + dependencies(tree, dapp, dname, trace+[(app,name)]) + ) + return deps + + +def remove_duplicates(l): + m = [] + for x in l: + if x not in m: + m.append(x) + return m + + +def needed_before_forwards(tree, app, name, sameapp=True): + """ + Returns a list of migrations that must be applied before (app, name), + in the order they should be applied. + Used to make sure a migration can be applied (and to help apply up to it). + """ + app_migrations = get_migration_names(app) + needed = [] + if sameapp: + for aname in app_migrations[:app_migrations.index(name)]: + needed += needed_before_forwards(tree, app, aname, False) + needed += [(app, aname)] + for dapp, dname in tree[app][name].needs: + needed += needed_before_forwards(tree, dapp, dname) + needed += [(dapp, dname)] + return remove_duplicates(needed) + + +def needed_before_backwards(tree, app, name, sameapp=True): + """ + Returns a list of migrations that must be unapplied before (app, name) is, + in the order they should be unapplied. + Used to make sure a migration can be unapplied (and to help unapply up to it). + """ + app_migrations = get_migration_names(app) + needed = [] + if sameapp: + for aname in reversed(app_migrations[app_migrations.index(name)+1:]): + needed += needed_before_backwards(tree, app, aname, False) + needed += [(app, aname)] + for dapp, dname in tree[app][name].needed_by: + needed += needed_before_backwards(tree, dapp, dname) + needed += [(dapp, dname)] + return remove_duplicates(needed) + + +def run_forwards(app, migrations, fake=False, silent=False): + """ + Runs the specified migrations forwards, in order. + """ + for migration in migrations: + app_name = get_app_name(app) + if not silent: + print " > %s: %s" % (app_name, migration) + klass = get_migration(app, migration) + if fake: + if not silent: + print " (faked)" + else: + db.start_transaction() + try: + klass().forwards() + db.execute_deferred_sql() + except: + db.rollback_transaction() + raise + else: + db.commit_transaction() + # Record us as having done this + record = MigrationHistory.for_migration(app_name, migration) + record.applied = datetime.datetime.utcnow() + record.save() + + +def run_backwards(app, migrations, ignore=[], fake=False, silent=False): + """ + Runs the specified migrations backwards, in order, skipping those + migrations in 'ignore'. + """ + for migration in migrations: + if migration not in ignore: + app_name = get_app_name(app) + if not silent: + print " < %s: %s" % (app_name, migration) + klass = get_migration(app, migration) + if fake: + if not silent: + print " (faked)" + else: + db.start_transaction() + try: + klass().backwards() + db.execute_deferred_sql() + except: + db.rollback_transaction() + raise + else: + db.commit_transaction() + # Record us as having not done this + record = MigrationHistory.for_migration(app_name, migration) + record.delete() + + +def right_side_of(x, y): + return left_side_of(reversed(x), reversed(y)) + + +def left_side_of(x, y): + return list(y)[:len(x)] == list(x) + + +def forwards_problems(tree, forwards, done, silent=False): + problems = [] + for app, name in forwards: + if (app, name) not in done: + for dapp, dname in needed_before_backwards(tree, app, name): + if (dapp, dname) in done: + if not silent: + print " ! Migration (%s, %s) should not have been applied before (%s, %s) but was." % (get_app_name(dapp), dname, get_app_name(app), name) + problems.append(((app, name), (dapp, dname))) + return problems + + + +def backwards_problems(tree, backwards, done, silent=False): + problems = [] + for app, name in backwards: + if (app, name) in done: + for dapp, dname in needed_before_forwards(tree, app, name): + if (dapp, dname) not in done: + if not silent: + print " ! Migration (%s, %s) should have been applied before (%s, %s) but wasn't." % (get_app_name(dapp), dname, get_app_name(app), name) + problems.append(((app, name), (dapp, dname))) + return problems + + +def migrate_app(app, target_name=None, resolve_mode=None, fake=False, yes=False, silent=False): + + app_name = get_app_name(app) + + db.debug = not silent + + # If any of their app names in the DB contain a ., they're 0.2 or below, so migrate em + longuns = MigrationHistory.objects.filter(app_name__contains=".") + if longuns: + for mh in longuns: + mh.app_name = short_from_long(mh.app_name) + mh.save() + if not silent: + print "- Updated your South 0.2 database." + + # Find out what delightful migrations we have + tree = dependency_tree() + migrations = get_migration_names(app) + + if target_name not in migrations and target_name not in ["zero", None]: + matches = [x for x in migrations if x.startswith(target_name)] + if len(matches) == 1: + target = migrations.index(matches[0]) + 1 + if not silent: + print " - Soft matched migration %s to %s." % ( + target_name, + matches[0] + ) + target_name = matches[0] + elif len(matches) > 1: + if not silent: + print " - Prefix %s matches more than one migration:" % target_name + print " " + "\n ".join(matches) + return + else: + if not silent: + print " ! '%s' is not a migration." % target_name + return + + # Check there's no strange ones in the database + ghost_migrations = [m for m in MigrationHistory.objects.filter(applied__isnull = False) if get_app(m.app_name) not in tree or m.migration not in tree[get_app(m.app_name)]] + if ghost_migrations: + if not silent: + print " ! These migrations are in the database but not on disk:" + print " - " + "\n - ".join(["%s: %s" % (x.app_name, x.migration) for x in ghost_migrations]) + print " ! I'm not trusting myself; fix this yourself by fiddling" + print " ! with the south_migrationhistory table." + return + + # Say what we're doing + if not silent: + print "Running migrations for %s:" % app_name + + # Get the forwards and reverse dependencies for this target + if target_name == None: + target_name = migrations[-1] + if target_name == "zero": + forwards = [] + backwards = needed_before_backwards(tree, app, migrations[0]) + [(app, migrations[0])] + else: + forwards = needed_before_forwards(tree, app, target_name) + [(app, target_name)] + # When migrating backwards we want to remove up to and including + # the next migration up in this app (not the next one, that includes other apps) + try: + migration_before_here = migrations[migrations.index(target_name)+1] + backwards = needed_before_backwards(tree, app, migration_before_here) + [(app, migration_before_here)] + except IndexError: + backwards = [] + + # Get the list of currently applied migrations from the db + current_migrations = [(get_app(m.app_name), m.migration) for m in MigrationHistory.objects.filter(applied__isnull = False)] + + direction = None + bad = False + + # Work out the direction + applied_for_this_app = list(MigrationHistory.objects.filter(app_name=app_name, applied__isnull=False).order_by("migration")) + if target_name == "zero": + direction = -1 + elif not applied_for_this_app: + direction = 1 + elif migrations.index(target_name) > migrations.index(applied_for_this_app[-1].migration): + direction = 1 + elif migrations.index(target_name) < migrations.index(applied_for_this_app[-1].migration): + direction = -1 + else: + direction = None + + # Is the whole forward branch applied? + missing = [step for step in forwards if step not in current_migrations] + # If they're all applied, we only know it's not backwards + if not missing: + direction = None + # If the remaining migrations are strictly a right segment of the forwards + # trace, we just need to go forwards to our target (and check for badness) + else: + problems = forwards_problems(tree, forwards, current_migrations, silent=silent) + if problems: + bad = True + direction = 1 + + # What about the whole backward trace then? + if not bad: + missing = [step for step in backwards if step not in current_migrations] + # If they're all missing, stick with the forwards decision + if missing == backwards: + pass + # If what's missing is a strict left segment of backwards (i.e. + # all the higher migrations) then we need to go backwards + else: + problems = backwards_problems(tree, backwards, current_migrations, silent=silent) + if problems: + bad = True + direction = -1 + + if bad and resolve_mode not in ['merge']: + if not silent: + print " ! Inconsistent migration history" + print " ! The following options are available:" + print " --merge: will just attempt the migration ignoring any potential dependency conflicts." + sys.exit(1) + + if direction == 1: + if not silent: + print " - Migrating forwards to %s." % target_name + for mapp, mname in forwards: + if (mapp, mname) not in current_migrations: + run_forwards(mapp, [mname], fake=fake, silent=silent) + elif direction == -1: + if not silent: + print " - Migrating backwards to just after %s." % target_name + for mapp, mname in backwards: + if (mapp, mname) in current_migrations: + run_backwards(mapp, [mname], fake=fake, silent=silent) + else: + if not silent: + print "- Nothing to migrate." \ No newline at end of file diff --git a/apps/south/models.py b/apps/south/models.py new file mode 100644 index 000000000..e95c79af7 --- /dev/null +++ b/apps/south/models.py @@ -0,0 +1,19 @@ +from django.db import models + +class MigrationHistory(models.Model): + app_name = models.CharField(max_length=255) + migration = models.CharField(max_length=255) + applied = models.DateTimeField(blank=True, null=True) + + @classmethod + def for_migration(cls, app_name, migration): + try: + return cls.objects.get( + app_name = app_name, + migration = migration, + ) + except cls.DoesNotExist: + return cls( + app_name = app_name, + migration = migration, + ) \ No newline at end of file diff --git a/apps/south/tests/__init__.py b/apps/south/tests/__init__.py new file mode 100644 index 000000000..d8953fefe --- /dev/null +++ b/apps/south/tests/__init__.py @@ -0,0 +1,11 @@ + +from django.conf import settings + +try: + skiptest = settings.SKIP_SOUTH_TESTS +except: + skiptest = False + +if not skiptest: + from south.tests.db import * + from south.tests.logic import * \ No newline at end of file diff --git a/apps/south/tests/db.py b/apps/south/tests/db.py new file mode 100644 index 000000000..c47f02198 --- /dev/null +++ b/apps/south/tests/db.py @@ -0,0 +1,83 @@ +import unittest + +from south.db import db +from django.db import connection, models + +# Create a list of error classes from the various database libraries +errors = [] +try: + from psycopg2 import ProgrammingError + errors.append(ProgrammingError) +except ImportError: + pass +errors = tuple(errors) + +class TestOperations(unittest.TestCase): + + """ + Tests if the various DB abstraction calls work. + Can only test a limited amount due to DB differences. + """ + + def setUp(self): + db.debug = False + + def test_create(self): + """ + Test creation and deletion of tables. + """ + cursor = connection.cursor() + # It needs to take at least 2 args + self.assertRaises(TypeError, db.create_table) + self.assertRaises(TypeError, db.create_table, "test1") + # Empty tables (i.e. no columns) are not fine, so make at least 1 + db.create_table("test1", [('email_confirmed', models.BooleanField(default=False))]) + db.start_transaction() + # And should exist + cursor.execute("SELECT * FROM test1") + # Make sure we can't do the same query on an empty table + try: + cursor.execute("SELECT * FROM nottheretest1") + self.fail("Non-existent table could be selected!") + except: + pass + # Clear the dirty transaction + db.rollback_transaction() + db.start_transaction() + # Remove the table + db.delete_table("test1") + # Make sure it went + try: + cursor.execute("SELECT * FROM test1") + self.fail("Just-deleted table could be selected!") + except: + pass + # Clear the dirty transaction + db.rollback_transaction() + db.start_transaction() + # Try deleting a nonexistent one + try: + db.delete_table("nottheretest1") + self.fail("Non-existent table could be deleted!") + except: + pass + db.rollback_transaction() + + def test_rename(self): + """ + Test column renaming + """ + cursor = connection.cursor() + db.create_table("test2", [('spam', models.BooleanField(default=False))]) + db.start_transaction() + # Make sure we can select the column + cursor.execute("SELECT spam FROM test2") + # Rename it + db.rename_column("test2", "spam", "eggs") + cursor.execute("SELECT eggs FROM test2") + try: + cursor.execute("SELECT spam FROM test2") + self.fail("Just-renamed column could be selected!") + except: + pass + db.rollback_transaction() \ No newline at end of file diff --git a/apps/south/tests/fakeapp/__init__.py b/apps/south/tests/fakeapp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/south/tests/fakeapp/migrations/0001_spam.py b/apps/south/tests/fakeapp/migrations/0001_spam.py new file mode 100644 index 000000000..d81454867 --- /dev/null +++ b/apps/south/tests/fakeapp/migrations/0001_spam.py @@ -0,0 +1,19 @@ +from south.db import db +from django.db import models + +class Migration: + + def forwards(self): + + # Model 'Spam' + db.create_table("southtest_spam", ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('weight', models.FloatField()), + ('expires', models.DateTimeField()), + ('name', models.CharField(max_length=255)) + )) + + def backwards(self): + + db.delete_table("southtest_spam") + diff --git a/apps/south/tests/fakeapp/migrations/0002_eggs.py b/apps/south/tests/fakeapp/migrations/0002_eggs.py new file mode 100644 index 000000000..3ec83999f --- /dev/null +++ b/apps/south/tests/fakeapp/migrations/0002_eggs.py @@ -0,0 +1,20 @@ +from south.db import db +from django.db import models + +class Migration: + + def forwards(self): + + Spam = db.mock_model(model_name='Spam', db_table='southtest_spam', db_tablespace='', pk_field_name='id', pk_field_type=models.AutoField) + + db.create_table("southtest_eggs", ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('size', models.FloatField()), + ('quantity', models.IntegerField()), + ('spam', models.ForeignKey(Spam)), + )) + + def backwards(self): + + db.delete_table("southtest_eggs") + diff --git a/apps/south/tests/fakeapp/migrations/__init__.py b/apps/south/tests/fakeapp/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/south/tests/fakeapp/models.py b/apps/south/tests/fakeapp/models.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/south/tests/logic.py b/apps/south/tests/logic.py new file mode 100644 index 000000000..dfb441eed --- /dev/null +++ b/apps/south/tests/logic.py @@ -0,0 +1,253 @@ +import unittest +import datetime +import sys +import os + +from south import migration + +# Add the tests directory so fakeapp is on sys.path +test_root = os.path.dirname(__file__) +sys.path.append(test_root) + + +class TestMigrationLogic(unittest.TestCase): + + """ + Tests if the various logic functions in migration actually work. + """ + + def create_fake_app(self, name): + + class Fake: + pass + + fake = Fake() + fake.__name__ = name + return fake + + + def create_test_app(self): + + class Fake: + pass + + fake = Fake() + fake.__name__ = "fakeapp.migrations" + fake.__file__ = os.path.join(test_root, "fakeapp", "migrations", "__init__.py") + return fake + + + def monkeypatch(self): + """Swaps out various Django calls for fake ones for our own nefarious purposes.""" + + def new_get_apps(): + return ['fakeapp'] + + from django.db import models + from django.conf import settings + models.get_apps_old, models.get_apps = models.get_apps, new_get_apps + settings.INSTALLED_APPS, settings.OLD_INSTALLED_APPS = ( + ["fakeapp"], + settings.INSTALLED_APPS, + ) + self.redo_app_cache() + setUp = monkeypatch + + + def unmonkeypatch(self): + """Undoes what monkeypatch did.""" + + from django.db import models + from django.conf import settings + models.get_apps = models.get_apps_old + settings.INSTALLED_APPS = settings.OLD_INSTALLED_APPS + self.redo_app_cache() + tearDown = unmonkeypatch + + + def redo_app_cache(self): + from django.db.models.loading import AppCache + a = AppCache() + a.loaded = False + a._populate() + + + def test_get_app_name(self): + self.assertEqual( + "southtest", + migration.get_app_name(self.create_fake_app("southtest.migrations")), + ) + self.assertEqual( + "baz", + migration.get_app_name(self.create_fake_app("foo.bar.baz.migrations")), + ) + + + def test_get_migrated_apps(self): + + P1 = __import__("fakeapp.migrations", {}, {}, ['']) + + self.assertEqual( + [P1], + list(migration.get_migrated_apps()), + ) + + + def test_get_app(self): + + P1 = __import__("fakeapp.migrations", {}, {}, ['']) + + self.assertEqual(P1, migration.get_app("fakeapp")) + self.assertEqual(P1, migration.get_app(self.create_fake_app("fakeapp.models"))) + + + def test_get_app_fullname(self): + self.assertEqual( + "southtest", + migration.get_app_fullname(self.create_fake_app("southtest.migrations")), + ) + self.assertEqual( + "foo.bar.baz", + migration.get_app_fullname(self.create_fake_app("foo.bar.baz.migrations")), + ) + + + def test_get_migration_names(self): + + app = self.create_test_app() + + self.assertEqual( + ["0001_spam", "0002_eggs"], + migration.get_migration_names(app), + ) + + + def test_get_migration_classes(self): + + app = self.create_test_app() + + # Can't use vanilla import, modules beginning with numbers aren't in grammar + M1 = __import__("fakeapp.migrations.0001_spam", {}, {}, ['Migration']).Migration + M2 = __import__("fakeapp.migrations.0002_eggs", {}, {}, ['Migration']).Migration + + self.assertEqual( + [M1, M2], + list(migration.get_migration_classes(app)), + ) + + + def test_get_migration(self): + + app = self.create_test_app() + + # Can't use vanilla import, modules beginning with numbers aren't in grammar + M1 = __import__("fakeapp.migrations.0001_spam", {}, {}, ['Migration']).Migration + M2 = __import__("fakeapp.migrations.0002_eggs", {}, {}, ['Migration']).Migration + + self.assertEqual(M1, migration.get_migration(app, "0001_spam")) + self.assertEqual(M2, migration.get_migration(app, "0002_eggs")) + + self.assertRaises(ValueError, migration.get_migration, app, "0001_jam") + + + def test_all_migrations(self): + + app = migration.get_app("fakeapp") + + self.assertEqual( + {app: { + "0001_spam": migration.get_migration(app, "0001_spam"), + "0002_eggs": migration.get_migration(app, "0002_eggs"), + }}, + migration.all_migrations(), + ) + + + def assertListEqual(self, list1, list2): + list1 = list(list1) + list2 = list(list2) + list1.sort() + list2.sort() + return self.assertEqual(list1, list2) + + + def test_apply_migrations(self): + + app = migration.get_app("fakeapp") + + # We should start with no migrations + self.assertEqual(list(migration.MigrationHistory.objects.all()), []) + + # Apply them normally + migration.migrate_app(app, target_name=None, resolve_mode=None, fake=False, silent=True) + + # We should finish with all migrations + self.assertListEqual( + ( + (u"fakeapp", u"0001_spam"), + (u"fakeapp", u"0002_eggs"), + ), + migration.MigrationHistory.objects.values_list("app_name", "migration"), + ) + + # Now roll them backwards + migration.migrate_app(app, target_name="zero", resolve_mode=None, fake=False, silent=True) + + # Finish with none + self.assertEqual(list(migration.MigrationHistory.objects.all()), []) + + + def test_migration_merge_forwards(self): + + app = migration.get_app("fakeapp") + + # We should start with no migrations + self.assertEqual(list(migration.MigrationHistory.objects.all()), []) + + # Insert one in the wrong order + migration.MigrationHistory.objects.create( + app_name = "fakeapp", + migration = "0002_eggs", + applied = datetime.datetime.now(), + ) + + # Did it go in? + self.assertListEqual( + ( + (u"fakeapp", u"0002_eggs"), + ), + migration.MigrationHistory.objects.values_list("app_name", "migration"), + ) + + # Apply them normally + try: + migration.migrate_app(app, target_name=None, resolve_mode=None, fake=False, silent=True) + except SystemExit: + pass + + # Nothing should have changed (no merge mode!) + self.assertListEqual( + ( + (u"fakeapp", u"0002_eggs"), + ), + migration.MigrationHistory.objects.values_list("app_name", "migration"), + ) + + # Apply with merge + migration.migrate_app(app, target_name=None, resolve_mode="merge", fake=False, silent=True) + + # We should finish with all migrations + self.assertListEqual( + ( + (u"fakeapp", u"0001_spam"), + (u"fakeapp", u"0002_eggs"), + ), + migration.MigrationHistory.objects.values_list("app_name", "migration"), + ) + + # Now roll them backwards + migration.migrate_app(app, target_name="0001", resolve_mode=None, fake=True, silent=True) + migration.migrate_app(app, target_name="zero", resolve_mode=None, fake=False, silent=True) + + # Finish with none + self.assertEqual(list(migration.MigrationHistory.objects.all()), []) \ No newline at end of file diff --git a/wolnelektury/settings.py b/wolnelektury/settings.py index 1e60e1d07..bdbc9923b 100644 --- a/wolnelektury/settings.py +++ b/wolnelektury/settings.py @@ -95,6 +95,7 @@ INSTALLED_APPS = [ 'django.contrib.admindocs', # external + 'south', 'newtagging', 'pagination', 'chunks', -- 2.20.1