From 93b01873677b426b379891faf95b9da0168b9991 Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Tue, 30 Jan 2024 12:38:34 -0500 Subject: [PATCH 01/23] Adds command to check for circular migration dependencies --- .gitignore | 2 + django_squash/db/migrations/loader.py | 5 +- django_squash/db/migrations/utils.py | 3 - .../management/commands/circularmigrations.py | 62 +++++++++++++++++++ 4 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 django_squash/management/commands/circularmigrations.py diff --git a/.gitignore b/.gitignore index ec1742d..e06a3f5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,7 @@ __pycache__ *.sqlite3 .eggs .coverage +coverage.xml +htmlcov build dist diff --git a/django_squash/db/migrations/loader.py b/django_squash/db/migrations/loader.py index 251be9d..1c76f54 100644 --- a/django_squash/db/migrations/loader.py +++ b/django_squash/db/migrations/loader.py @@ -10,7 +10,10 @@ class SquashMigrationLoader(MigrationLoader): def __init__(self, *args, **kwargs): - original_migration_modules = settings.MIGRATION_MODULES.copy() + # keep a copy of the original migration modules to restore it later + original_migration_modules = settings.MIGRATION_MODULES + # make a copy of the migration modules so we can modify it + settings.MIGRATION_MODULES = settings.MIGRATION_MODULES.copy() project_path = os.path.abspath(os.curdir) with ExitStack() as stack: diff --git a/django_squash/db/migrations/utils.py b/django_squash/db/migrations/utils.py index d30a587..d99c29f 100644 --- a/django_squash/db/migrations/utils.py +++ b/django_squash/db/migrations/utils.py @@ -192,15 +192,12 @@ def replace_migration_attribute(source, attr, value): # Remove the lines that form the multi-line "replaces" statement. p_count = 0 b_count = 0 - col_offset = None - name = None output = [] for lineno, line in enumerate(source.splitlines()): if lineno + 1 in comment_out_nodes.keys(): output.append(" " * comment_out_nodes[lineno + 1][0] + attr + " = " + str(value)) p_count = 0 b_count = 0 - col_offset, name = comment_out_nodes[lineno + 1] p_count, b_count = find_brackets(line, p_count, b_count) elif p_count != 0 or b_count != 0: p_count, b_count = find_brackets(line, p_count, b_count) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py new file mode 100644 index 0000000..4a475b3 --- /dev/null +++ b/django_squash/management/commands/circularmigrations.py @@ -0,0 +1,62 @@ +import itertools +import os + +from django.conf import settings +from django.apps import apps +from django.core.management.base import BaseCommand, CommandError, no_translations +from django.db.migrations.loader import MigrationLoader +from django.db.migrations.state import ProjectState + +from django_squash import settings as app_settings +from django_squash.db.migrations import serializer +from django_squash.db.migrations.autodetector import SquashMigrationAutodetector +from django_squash.db.migrations.loader import SquashMigrationLoader +from django_squash.db.migrations.questioner import NonInteractiveMigrationQuestioner +from django_squash.db.migrations.writer import MigrationWriter + + +class Command(BaseCommand): + def add_arguments(self, parser): + pass + + @no_translations + def handle(self, **kwargs): + self.verbosity = 1 + self.include_header = False + + dependency_list = settings.INSTALLED_APPS + apps.all_models + + questioner = NonInteractiveMigrationQuestioner(specified_apps=None, dry_run=False) + squash_loader = SquashMigrationLoader(None, ignore_no_migrations=True) + autodetector = SquashMigrationAutodetector( + squash_loader.project_state(), + ProjectState.from_apps(apps), + questioner, + ) + + changes = autodetector.changes(squash_loader.graph, trim_to_apps=None, convert_apps=None, migration_name=None) + for app_label, migrations in changes.items(): + app_label = apps.get_app_config(app_label).name + try: + app_ranking = dependency_list.index(app_label) + except ValueError: + print(f"{app_label} not found") + continue + for migration in migrations: + for depends_on_app_label, _ in migration.dependencies: + if depends_on_app_label == "__setting__": + continue + depends_on_app_label = apps.get_app_config(depends_on_app_label).name + depends_on_app_ranking = dependency_list.index(depends_on_app_label) + if depends_on_app_ranking > app_ranking: + print(f"{app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})") + # elif depends_on_app_ranking < app_ranking: + # print(f"{app_label} ({app_ranking}) < {depends_on_app_label} ({depends_on_app_ranking})") + + + + import ipdb; print('\a'); ipdb.sset_trace() + a=a + + From 0ae7572025e42d0b6fed908341f959fec587e31b Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Fri, 1 Mar 2024 16:28:11 -0500 Subject: [PATCH 02/23] . --- .../management/commands/circularmigrations.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index 4a475b3..d434b9b 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -44,19 +44,30 @@ def handle(self, **kwargs): print(f"{app_label} not found") continue for migration in migrations: + bad_dependencies = [] for depends_on_app_label, _ in migration.dependencies: if depends_on_app_label == "__setting__": continue depends_on_app_label = apps.get_app_config(depends_on_app_label).name depends_on_app_ranking = dependency_list.index(depends_on_app_label) if depends_on_app_ranking > app_ranking: - print(f"{app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})") + print(f"* {app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})") + bad_dependencies.append((app_label, depends_on_app_label)) + + if not bad_dependencies: + continue + + import ipdb; print('\a'); ipdb.sset_trace() + for operation in migration.operations: + for field in operation.fields: + continue + + # elif depends_on_app_ranking < app_ranking: # print(f"{app_label} ({app_ranking}) < {depends_on_app_label} ({depends_on_app_ranking})") - import ipdb; print('\a'); ipdb.sset_trace() a=a From c562576af2bff6abd0afe2f46279c43d0db9f2de Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Fri, 1 Mar 2024 17:05:00 -0500 Subject: [PATCH 03/23] Doesnt crash --- .../management/commands/circularmigrations.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index d434b9b..6fcea67 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -52,22 +52,22 @@ def handle(self, **kwargs): depends_on_app_ranking = dependency_list.index(depends_on_app_label) if depends_on_app_ranking > app_ranking: print(f"* {app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})") - bad_dependencies.append((app_label, depends_on_app_label)) + bad_dependencies.append(depends_on_app_label.split('.')[-1]) if not bad_dependencies: continue - import ipdb; print('\a'); ipdb.sset_trace() + references_found = [] for operation in migration.operations: - for field in operation.fields: - continue - - - # elif depends_on_app_ranking < app_ranking: - # print(f"{app_label} ({app_ranking}) < {depends_on_app_label} ({depends_on_app_ranking})") - - - - a=a - - + if hasattr(operation, "field") and hasattr(operation.field, "related_model"): + if operation.field.related_model._meta.app_label in bad_dependencies: + references_found.append((operation.model_name, operation.name, operation.field.related_model)) + + if hasattr(operation, "fields"): + for name, field in operation.fields: + if hasattr(field, "related_model") and field.related_model and not isinstance(field.related_model, str): + if field.related_model._meta.app_label in bad_dependencies: + references_found.append((operation.name, name, field.related_model)) + + for model_name, field_name, model in references_found: + print(f" references {model._meta.app_label}.{model._meta.object_name} via {model_name}.{field_name}") From 199868ca270f6a7cd3c6f897026ce9c27a2e8527 Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Fri, 1 Mar 2024 17:07:06 -0500 Subject: [PATCH 04/23] Clean up --- .../management/commands/circularmigrations.py | 68 +++++++++++++------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index 6fcea67..771f8df 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -1,18 +1,11 @@ -import itertools -import os - -from django.conf import settings from django.apps import apps -from django.core.management.base import BaseCommand, CommandError, no_translations -from django.db.migrations.loader import MigrationLoader +from django.conf import settings +from django.core.management.base import BaseCommand, no_translations from django.db.migrations.state import ProjectState -from django_squash import settings as app_settings -from django_squash.db.migrations import serializer from django_squash.db.migrations.autodetector import SquashMigrationAutodetector from django_squash.db.migrations.loader import SquashMigrationLoader from django_squash.db.migrations.questioner import NonInteractiveMigrationQuestioner -from django_squash.db.migrations.writer import MigrationWriter class Command(BaseCommand): @@ -27,7 +20,9 @@ def handle(self, **kwargs): dependency_list = settings.INSTALLED_APPS apps.all_models - questioner = NonInteractiveMigrationQuestioner(specified_apps=None, dry_run=False) + questioner = NonInteractiveMigrationQuestioner( + specified_apps=None, dry_run=False + ) squash_loader = SquashMigrationLoader(None, ignore_no_migrations=True) autodetector = SquashMigrationAutodetector( squash_loader.project_state(), @@ -35,7 +30,12 @@ def handle(self, **kwargs): questioner, ) - changes = autodetector.changes(squash_loader.graph, trim_to_apps=None, convert_apps=None, migration_name=None) + changes = autodetector.changes( + squash_loader.graph, + trim_to_apps=None, + convert_apps=None, + migration_name=None, + ) for app_label, migrations in changes.items(): app_label = apps.get_app_config(app_label).name try: @@ -48,26 +48,52 @@ def handle(self, **kwargs): for depends_on_app_label, _ in migration.dependencies: if depends_on_app_label == "__setting__": continue - depends_on_app_label = apps.get_app_config(depends_on_app_label).name + depends_on_app_label = apps.get_app_config( + depends_on_app_label + ).name depends_on_app_ranking = dependency_list.index(depends_on_app_label) if depends_on_app_ranking > app_ranking: - print(f"* {app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})") - bad_dependencies.append(depends_on_app_label.split('.')[-1]) + print( + f"* {app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})" + ) + bad_dependencies.append(depends_on_app_label.split(".")[-1]) if not bad_dependencies: continue references_found = [] for operation in migration.operations: - if hasattr(operation, "field") and hasattr(operation.field, "related_model"): - if operation.field.related_model._meta.app_label in bad_dependencies: - references_found.append((operation.model_name, operation.name, operation.field.related_model)) + if hasattr(operation, "field") and hasattr( + operation.field, "related_model" + ): + if ( + operation.field.related_model._meta.app_label + in bad_dependencies + ): + references_found.append( + ( + operation.model_name, + operation.name, + operation.field.related_model, + ) + ) if hasattr(operation, "fields"): for name, field in operation.fields: - if hasattr(field, "related_model") and field.related_model and not isinstance(field.related_model, str): - if field.related_model._meta.app_label in bad_dependencies: - references_found.append((operation.name, name, field.related_model)) + if ( + hasattr(field, "related_model") + and field.related_model + and not isinstance(field.related_model, str) + ): + if ( + field.related_model._meta.app_label + in bad_dependencies + ): + references_found.append( + (operation.name, name, field.related_model) + ) for model_name, field_name, model in references_found: - print(f" references {model._meta.app_label}.{model._meta.object_name} via {model_name}.{field_name}") + print( + f" references {model._meta.app_label}.{model._meta.object_name} via {model_name}.{field_name}" + ) From 84b279128fd5440707507187fd3a012da433433f Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Fri, 1 Mar 2024 20:39:50 -0500 Subject: [PATCH 05/23] Fixes ignore apps --- .../management/commands/squash_migrations.py | 29 +++++++++---------- tests/test_migrations.py | 14 +++++++-- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/django_squash/management/commands/squash_migrations.py b/django_squash/management/commands/squash_migrations.py index 3ab9044..c7ad5d1 100644 --- a/django_squash/management/commands/squash_migrations.py +++ b/django_squash/management/commands/squash_migrations.py @@ -16,10 +16,9 @@ class Command(BaseCommand): def add_arguments(self, parser): - parser.add_argument("--only", action="append", nargs="*", help="Only squash the specified apps") + parser.add_argument("--only", nargs="*", help="Only squash the specified apps") parser.add_argument( "--ignore-app", - action="append", nargs="*", default=app_settings.DJANGO_SQUASH_IGNORE_APPS, help="Ignore app name from quashing, ensure that there is nothing dependent on these apps. " @@ -47,24 +46,22 @@ def handle(self, **kwargs): ignore_apps = [] bad_apps = [] - for app_labels in kwargs["ignore_app"]: - for app_label in app_labels: - try: - apps.get_app_config(app_label) - ignore_apps.append(app_label) - except (LookupError, TypeError): - bad_apps.append(str(app_label)) + for app_label in kwargs["ignore_app"]: + try: + apps.get_app_config(app_label) + ignore_apps.append(app_label) + except (LookupError, TypeError): + bad_apps.append(str(app_label)) if kwargs["only"]: only_apps = [] - for app_labels in kwargs["only"]: - for app_label in app_labels: - try: - apps.get_app_config(app_label) - only_apps.append(app_label) - except (LookupError, TypeError): - bad_apps.append(str(app_label)) + for app_label in kwargs["only"]: + try: + apps.get_app_config(app_label) + only_apps.append(app_label) + except (LookupError, TypeError): + bad_apps.append(app_label) for app_name in apps.app_configs.keys(): if app_name not in only_apps: diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 45c41ff..a08cad6 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -244,10 +244,18 @@ def test_invalid_apps(call_squash_migrations): with pytest.raises(CommandError) as error: call_squash_migrations( "--ignore-app", - "a", - "b", + "aaa", + "bbb", ) - assert str(error.value) == "The following apps are not valid: a, b" + assert str(error.value) == "The following apps are not valid: aaa, bbb" + + +@pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") +def test_invalid_apps_ignore(monkeypatch, call_squash_migrations): + monkeypatch.setattr('django_squash.settings.DJANGO_SQUASH_IGNORE_APPS', ["aaa", "bbb"]) + with pytest.raises(CommandError) as error: + call_squash_migrations() + assert str(error.value) == "The following apps are not valid: aaa, bbb" @pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") From 74c7f80eedf9bd954ac2f57ab6054d35ed83ce6f Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Fri, 1 Mar 2024 20:45:57 -0500 Subject: [PATCH 06/23] linter --- .../management/commands/circularmigrations.py | 30 +++++-------------- tests/test_migrations.py | 17 ++++++++--- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index 771f8df..0483825 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -20,9 +20,7 @@ def handle(self, **kwargs): dependency_list = settings.INSTALLED_APPS apps.all_models - questioner = NonInteractiveMigrationQuestioner( - specified_apps=None, dry_run=False - ) + questioner = NonInteractiveMigrationQuestioner(specified_apps=None, dry_run=False) squash_loader = SquashMigrationLoader(None, ignore_no_migrations=True) autodetector = SquashMigrationAutodetector( squash_loader.project_state(), @@ -48,14 +46,10 @@ def handle(self, **kwargs): for depends_on_app_label, _ in migration.dependencies: if depends_on_app_label == "__setting__": continue - depends_on_app_label = apps.get_app_config( - depends_on_app_label - ).name + depends_on_app_label = apps.get_app_config(depends_on_app_label).name depends_on_app_ranking = dependency_list.index(depends_on_app_label) if depends_on_app_ranking > app_ranking: - print( - f"* {app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})" - ) + print(f"* {app_label} ({app_ranking}) > {depends_on_app_label} ({depends_on_app_ranking})") bad_dependencies.append(depends_on_app_label.split(".")[-1]) if not bad_dependencies: @@ -63,13 +57,8 @@ def handle(self, **kwargs): references_found = [] for operation in migration.operations: - if hasattr(operation, "field") and hasattr( - operation.field, "related_model" - ): - if ( - operation.field.related_model._meta.app_label - in bad_dependencies - ): + if hasattr(operation, "field") and hasattr(operation.field, "related_model"): + if operation.field.related_model._meta.app_label in bad_dependencies: references_found.append( ( operation.model_name, @@ -85,13 +74,8 @@ def handle(self, **kwargs): and field.related_model and not isinstance(field.related_model, str) ): - if ( - field.related_model._meta.app_label - in bad_dependencies - ): - references_found.append( - (operation.name, name, field.related_model) - ) + if field.related_model._meta.app_label in bad_dependencies: + references_found.append((operation.name, name, field.related_model)) for model_name, field_name, model in references_found: print( diff --git a/tests/test_migrations.py b/tests/test_migrations.py index a08cad6..e2ff561 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -252,7 +252,7 @@ def test_invalid_apps(call_squash_migrations): @pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") def test_invalid_apps_ignore(monkeypatch, call_squash_migrations): - monkeypatch.setattr('django_squash.settings.DJANGO_SQUASH_IGNORE_APPS', ["aaa", "bbb"]) + monkeypatch.setattr("django_squash.settings.DJANGO_SQUASH_IGNORE_APPS", ["aaa", "bbb"]) with pytest.raises(CommandError) as error: call_squash_migrations() assert str(error.value) == "The following apps are not valid: aaa, bbb" @@ -262,7 +262,10 @@ def test_invalid_apps_ignore(monkeypatch, call_squash_migrations): def test_ignore_apps_argument(call_squash_migrations, monkeypatch): mock_squash = unittest.mock.MagicMock() - monkeypatch.setattr("django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", mock_squash) + monkeypatch.setattr( + "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", + mock_squash, + ) with pytest.raises(CommandError) as error: call_squash_migrations( "--ignore-app", @@ -278,7 +281,10 @@ def test_ignore_apps_argument(call_squash_migrations, monkeypatch): def test_only_argument(call_squash_migrations, settings, monkeypatch): mock_squash = unittest.mock.MagicMock() - monkeypatch.setattr("django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", mock_squash) + monkeypatch.setattr( + "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", + mock_squash, + ) with pytest.raises(CommandError) as error: call_squash_migrations( "--only", @@ -294,7 +300,10 @@ def test_only_argument(call_squash_migrations, settings, monkeypatch): def test_only_argument_with_invalid_apps(call_squash_migrations, monkeypatch): mock_squash = unittest.mock.MagicMock() - monkeypatch.setattr("django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", mock_squash) + monkeypatch.setattr( + "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", + mock_squash, + ) with pytest.raises(CommandError) as error: call_squash_migrations( "--only", From f254ae4b197f177939cbde3be903f0872a893f0f Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Fri, 1 Mar 2024 21:12:39 -0500 Subject: [PATCH 07/23] Adds ignore app --- django_squash/db/migrations/autodetector.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/django_squash/db/migrations/autodetector.py b/django_squash/db/migrations/autodetector.py index 6a97827..8aa3e55 100644 --- a/django_squash/db/migrations/autodetector.py +++ b/django_squash/db/migrations/autodetector.py @@ -129,7 +129,7 @@ def rename_migrations(self, original, graph, changes, migration_name): migration_name or "squashed", ) - def convert_migration_references_to_objects(self, original, graph, changes): + def convert_migration_references_to_objects(self, original, graph, changes, ignore_apps): """ Swap django.db.migrations.Migration with a custom one that behaves like a tuple when read, but is still an object for the purpose of easy renames. @@ -151,6 +151,10 @@ def convert_migration_references_to_objects(self, original, graph, changes): for migration in migrations: new_dependencies = [] for dependency in migration.dependencies: + if migration_id[0] in ignore_apps: + new_dependencies.append(original.graph.leaf_nodes(migration_id[0])[0]) + continue + if dependency[0] == "__setting__": app_label = getattr(settings, dependency[1]).split(".")[0] migrations = [ @@ -200,7 +204,7 @@ def squash(self, real_loader, squash_loader, ignore_apps=None, migration_name=No changes.pop(app, None) self.create_deleted_models_migrations(real_loader, changes) - self.convert_migration_references_to_objects(real_loader, graph, changes) + self.convert_migration_references_to_objects(real_loader, graph, changes, ignore_apps) self.rename_migrations(real_loader, graph, changes, migration_name) self.replace_current_migrations(real_loader, graph, changes) self.add_non_elidables(real_loader, squash_loader, changes) From 0c5689b2402fd3bf8106f2e99c05f02c15eedfb8 Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Fri, 1 Mar 2024 21:36:30 -0500 Subject: [PATCH 08/23] . --- django_squash/db/migrations/autodetector.py | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/django_squash/db/migrations/autodetector.py b/django_squash/db/migrations/autodetector.py index 8aa3e55..61857f9 100644 --- a/django_squash/db/migrations/autodetector.py +++ b/django_squash/db/migrations/autodetector.py @@ -151,12 +151,13 @@ def convert_migration_references_to_objects(self, original, graph, changes, igno for migration in migrations: new_dependencies = [] for dependency in migration.dependencies: - if migration_id[0] in ignore_apps: + dep_app_label, dep_migration = dependency + if dep_app_label in ignore_apps: new_dependencies.append(original.graph.leaf_nodes(migration_id[0])[0]) continue - if dependency[0] == "__setting__": - app_label = getattr(settings, dependency[1]).split(".")[0] + if dep_app_label == "__setting__": + app_label = getattr(settings, dep_migration).split(".")[0] migrations = [ migration for (app, _), migration in migrations_by_name.items() if app == app_label ] @@ -166,10 +167,10 @@ def convert_migration_references_to_objects(self, original, graph, changes, igno # Leave as is, the django's migration writer will handle this by default new_dependencies.append(dependency) continue - elif dependency[1] == "__first__": - dependency = original.graph.root_nodes(dependency[0])[0] - elif dependency[1] == "__latest__": - dependency = original.graph.leaf_nodes(dependency[0])[0] + elif dep_migration == "__first__": + dependency = original.graph.root_nodes(dep_app_label)[0] + elif dep_migration == "__latest__": + dependency = original.graph.leaf_nodes(dep_app_label)[0] migration_id = dependency if migration_id not in migrations_by_name: @@ -194,7 +195,7 @@ def create_deleted_models_migrations(self, loader, changes): instance.replaces = migrations changes[app_label] = [instance] - def squash(self, real_loader, squash_loader, ignore_apps=None, migration_name=None): + def squash(self, real_loader, squash_loader, ignore_apps, migration_name=None): changes_ = self.delete_old_squashed(real_loader, ignore_apps) graph = squash_loader.graph @@ -214,12 +215,13 @@ def squash(self, real_loader, squash_loader, ignore_apps=None, migration_name=No return changes - def delete_old_squashed(self, loader, ignore_apps=None): + def delete_old_squashed(self, loader, ignore_apps): changes = defaultdict(set) project_path = os.path.abspath(os.curdir) project_apps = [ app.label for app in apps.get_app_configs() if utils.source_directory(app.module).startswith(project_path) ] + import ipdb; print('\a'); ipdb.sset_trace() real_migrations = ( Migration.from_migration(loader.disk_migrations[key]) for key in loader.graph.node_map.keys() @@ -227,14 +229,14 @@ def delete_old_squashed(self, loader, ignore_apps=None): project_migrations = [ migration for migration in real_migrations - if migration.app_label in project_apps and migration.app_label not in ignore_apps or [] + if migration.app_label in project_apps and migration.app_label not in ignore_apps ] replaced_migrations = [ Migration.from_migration(migration) for migration in project_migrations if migration.replaces ] migrations_to_remove = set() - for migration in (y for x in replaced_migrations for y in x.replaces if y[0] not in ignore_apps or []): + for migration in (y for x in replaced_migrations for y in x.replaces if y[0] not in ignore_apps): real_migration = Migration.from_migration(loader.disk_migrations[migration]) real_migration._deleted = True migrations_to_remove.add(migration) From 24fa8b311c49f944bba1911396cb4579c9328d14 Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Sat, 2 Mar 2024 04:18:44 -0500 Subject: [PATCH 09/23] If there is a previously ignored app, we remove it if --only is used --- django_squash/db/migrations/autodetector.py | 13 +++--------- .../management/commands/squash_migrations.py | 3 +++ tests/test_migrations.py | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/django_squash/db/migrations/autodetector.py b/django_squash/db/migrations/autodetector.py index 61857f9..5c6c0f8 100644 --- a/django_squash/db/migrations/autodetector.py +++ b/django_squash/db/migrations/autodetector.py @@ -221,16 +221,9 @@ def delete_old_squashed(self, loader, ignore_apps): project_apps = [ app.label for app in apps.get_app_configs() if utils.source_directory(app.module).startswith(project_path) ] - import ipdb; print('\a'); ipdb.sset_trace() - - real_migrations = ( - Migration.from_migration(loader.disk_migrations[key]) for key in loader.graph.node_map.keys() - ) - project_migrations = [ - migration - for migration in real_migrations - if migration.app_label in project_apps and migration.app_label not in ignore_apps - ] + + real_migrations = [Migration.from_migration(loader.disk_migrations[key]) for key in loader.graph.node_map.keys()] + project_migrations = [migration for migration in real_migrations if migration.app_label in project_apps and migration.app_label not in ignore_apps] replaced_migrations = [ Migration.from_migration(migration) for migration in project_migrations if migration.replaces ] diff --git a/django_squash/management/commands/squash_migrations.py b/django_squash/management/commands/squash_migrations.py index c7ad5d1..766e1f0 100644 --- a/django_squash/management/commands/squash_migrations.py +++ b/django_squash/management/commands/squash_migrations.py @@ -60,6 +60,9 @@ def handle(self, **kwargs): try: apps.get_app_config(app_label) only_apps.append(app_label) + # Edge case: if the app was previously ignored, remove it from the ignore list + if app_label in ignore_apps: + ignore_apps.remove(app_label) except (LookupError, TypeError): bad_apps.append(app_label) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index e2ff561..b2b9859 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -258,6 +258,27 @@ def test_invalid_apps_ignore(monkeypatch, call_squash_migrations): assert str(error.value) == "The following apps are not valid: aaa, bbb" +@pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") +def test_only_apps_with_ignored_app(call_squash_migrations, monkeypatch): + + mock_squash = unittest.mock.MagicMock() + monkeypatch.setattr( + "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", + mock_squash, + ) + with pytest.raises(CommandError) as error: + call_squash_migrations( + "--ignore-app", + "app2", + "app", + "--only", + "app2", + "app", + ) + assert mock_squash.called + assert set(mock_squash.call_args[1]["ignore_apps"]) == {'django_squash', 'app3'} + + @pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") def test_ignore_apps_argument(call_squash_migrations, monkeypatch): From 821126e506e5bdf63b328fa67e8ffcc288673e0f Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Sun, 3 Mar 2024 08:00:50 -0500 Subject: [PATCH 10/23] . --- README.rst | 2 +- tests/test_migrations.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c03122f..79027c9 100644 --- a/README.rst +++ b/README.rst @@ -60,7 +60,7 @@ Developing .. code-block:: shell - docker run --rm -it -v .:/app -e PYTHONDONTWRITEBYTECODE=1 python:3.12 bash -c 'cd app; pip install -e .[test]; echo; echo; echo "run **pytest** to run tests"; echo; exec bash' + docker run --rm -it -v .:/app -v django-squash-pip-cache:/root/.cache/pip -e PYTHONDONTWRITEBYTECODE=1 python:3.12 bash -c "cd app; pip install -e .[test]; echo \"alias linters=\\\"git diff origin/master --name-only | grep '\.py$' | tee >(xargs black --config .black) >(xargs isort)\\\"\" >> ~/.bash_profile; printf '\n\n\nrun **pytest** to run tests, **linters** to run linters\n\n'; exec bash --init-file ~/.bash_profile" Alternatively, you can also create a virtual environment and run diff --git a/tests/test_migrations.py b/tests/test_migrations.py index b2b9859..b7f77da 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -260,7 +260,9 @@ def test_invalid_apps_ignore(monkeypatch, call_squash_migrations): @pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") def test_only_apps_with_ignored_app(call_squash_migrations, monkeypatch): - + """ + Edge case: if the app was previously ignored, remove it from the ignore list + """ mock_squash = unittest.mock.MagicMock() monkeypatch.setattr( "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", @@ -275,6 +277,7 @@ def test_only_apps_with_ignored_app(call_squash_migrations, monkeypatch): "app2", "app", ) + assert str(error.value) == "There are no migrations to squash." assert mock_squash.called assert set(mock_squash.call_args[1]["ignore_apps"]) == {'django_squash', 'app3'} From 06524aff53787cfbf21ef8f8b1d26fcb349b8e14 Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Sun, 3 Mar 2024 08:03:34 -0500 Subject: [PATCH 11/23] linter --- django_squash/db/migrations/autodetector.py | 10 ++++++++-- tests/test_migrations.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/django_squash/db/migrations/autodetector.py b/django_squash/db/migrations/autodetector.py index 5c6c0f8..c190b1d 100644 --- a/django_squash/db/migrations/autodetector.py +++ b/django_squash/db/migrations/autodetector.py @@ -222,8 +222,14 @@ def delete_old_squashed(self, loader, ignore_apps): app.label for app in apps.get_app_configs() if utils.source_directory(app.module).startswith(project_path) ] - real_migrations = [Migration.from_migration(loader.disk_migrations[key]) for key in loader.graph.node_map.keys()] - project_migrations = [migration for migration in real_migrations if migration.app_label in project_apps and migration.app_label not in ignore_apps] + real_migrations = [ + Migration.from_migration(loader.disk_migrations[key]) for key in loader.graph.node_map.keys() + ] + project_migrations = [ + migration + for migration in real_migrations + if migration.app_label in project_apps and migration.app_label not in ignore_apps + ] replaced_migrations = [ Migration.from_migration(migration) for migration in project_migrations if migration.replaces ] diff --git a/tests/test_migrations.py b/tests/test_migrations.py index b7f77da..1eeb501 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -279,7 +279,7 @@ def test_only_apps_with_ignored_app(call_squash_migrations, monkeypatch): ) assert str(error.value) == "There are no migrations to squash." assert mock_squash.called - assert set(mock_squash.call_args[1]["ignore_apps"]) == {'django_squash', 'app3'} + assert set(mock_squash.call_args[1]["ignore_apps"]) == {"django_squash", "app3"} @pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") From db6a1776eca553f17f3d953f4f778ac9e5f4f685 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Thu, 21 Mar 2024 11:04:42 +0100 Subject: [PATCH 12/23] Fixes test --- tests/test_migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index ca01641..a48531b 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -320,7 +320,7 @@ def test_only_apps_with_ignored_app(call_squash_migrations, monkeypatch): ) assert str(error.value) == "There are no migrations to squash." assert mock_squash.called - assert set(mock_squash.call_args[1]["ignore_apps"]) == {"django_squash", "app3"} + assert set(mock_squash.call_args[1]["ignore_apps"]) == {"auth", "contenttypes", "django_squash", "app3"} @pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") From 15e5b4869180781f8375744e652d00f5a84dbb30 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Thu, 21 Mar 2024 11:10:01 +0100 Subject: [PATCH 13/23] Going back to master.. --- django_squash/db/migrations/autodetector.py | 25 +++++------- .../management/commands/squash_migrations.py | 3 -- tests/test_migrations.py | 39 ++----------------- 3 files changed, 13 insertions(+), 54 deletions(-) diff --git a/django_squash/db/migrations/autodetector.py b/django_squash/db/migrations/autodetector.py index 6368a43..19aa625 100644 --- a/django_squash/db/migrations/autodetector.py +++ b/django_squash/db/migrations/autodetector.py @@ -158,13 +158,8 @@ def convert_migration_references_to_objects(self, original, changes, ignore_apps for migration in migrations: new_dependencies = [] for dependency in migration.dependencies: - dep_app_label, dep_migration = dependency - if dep_app_label in ignore_apps: - new_dependencies.append(original.graph.leaf_nodes(migration_id[0])[0]) - continue - - if dep_app_label == "__setting__": - app_label = getattr(settings, dep_migration).split(".")[0] + if dependency[0] == "__setting__": + app_label = getattr(settings, dependency[1]).split(".")[0] migrations = [ migration for (app, _), migration in migrations_by_name.items() if app == app_label ] @@ -174,10 +169,10 @@ def convert_migration_references_to_objects(self, original, changes, ignore_apps # Leave as is, the django's migration writer will handle this by default new_dependencies.append(dependency) continue - elif dep_migration == "__first__": - dependency = original.graph.root_nodes(dep_app_label)[0] - elif dep_migration == "__latest__": - dependency = original.graph.leaf_nodes(dep_app_label)[0] + elif dependency[1] == "__first__": + dependency = original.graph.root_nodes(dependency[0])[0] + elif dependency[1] == "__latest__": + dependency = original.graph.leaf_nodes(dependency[0])[0] migration_id = dependency if migration_id not in migrations_by_name: @@ -229,20 +224,20 @@ def delete_old_squashed(self, loader, ignore_apps): app.label for app in apps.get_app_configs() if utils.source_directory(app.module).startswith(project_path) ] - real_migrations = [ + real_migrations = ( Migration.from_migration(loader.disk_migrations[key]) for key in loader.graph.node_map.keys() - ] + ) project_migrations = [ migration for migration in real_migrations - if migration.app_label in project_apps and migration.app_label not in ignore_apps + if migration.app_label in project_apps and migration.app_label not in ignore_apps or [] ] replaced_migrations = [ Migration.from_migration(migration) for migration in project_migrations if migration.replaces ] migrations_to_remove = set() - for migration in (y for x in replaced_migrations for y in x.replaces if y[0] not in ignore_apps): + for migration in (y for x in replaced_migrations for y in x.replaces if y[0] not in ignore_apps or []): real_migration = Migration.from_migration(loader.disk_migrations[migration]) real_migration._deleted = True migrations_to_remove.add(migration) diff --git a/django_squash/management/commands/squash_migrations.py b/django_squash/management/commands/squash_migrations.py index 766e1f0..c7ad5d1 100644 --- a/django_squash/management/commands/squash_migrations.py +++ b/django_squash/management/commands/squash_migrations.py @@ -60,9 +60,6 @@ def handle(self, **kwargs): try: apps.get_app_config(app_label) only_apps.append(app_label) - # Edge case: if the app was previously ignored, remove it from the ignore list - if app_label in ignore_apps: - ignore_apps.remove(app_label) except (LookupError, TypeError): bad_apps.append(app_label) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index a48531b..bcf08c1 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -299,38 +299,11 @@ def test_invalid_apps_ignore(monkeypatch, call_squash_migrations): assert str(error.value) == "The following apps are not valid: aaa, bbb" -@pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") -def test_only_apps_with_ignored_app(call_squash_migrations, monkeypatch): - """ - Edge case: if the app was previously ignored, remove it from the ignore list - """ - mock_squash = unittest.mock.MagicMock() - monkeypatch.setattr( - "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", - mock_squash, - ) - with pytest.raises(CommandError) as error: - call_squash_migrations( - "--ignore-app", - "app2", - "app", - "--only", - "app2", - "app", - ) - assert str(error.value) == "There are no migrations to squash." - assert mock_squash.called - assert set(mock_squash.call_args[1]["ignore_apps"]) == {"auth", "contenttypes", "django_squash", "app3"} - - @pytest.mark.temporary_migration_module(module="app.test_empty", app_label="app") def test_ignore_apps_argument(call_squash_migrations, monkeypatch): mock_squash = unittest.mock.MagicMock() - monkeypatch.setattr( - "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", - mock_squash, - ) + monkeypatch.setattr("django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", mock_squash) with pytest.raises(CommandError) as error: call_squash_migrations( "--ignore-app", @@ -346,10 +319,7 @@ def test_ignore_apps_argument(call_squash_migrations, monkeypatch): def test_only_argument(call_squash_migrations, settings, monkeypatch): mock_squash = unittest.mock.MagicMock() - monkeypatch.setattr( - "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", - mock_squash, - ) + monkeypatch.setattr("django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", mock_squash) with pytest.raises(CommandError) as error: call_squash_migrations( "--only", @@ -366,10 +336,7 @@ def test_only_argument(call_squash_migrations, settings, monkeypatch): def test_only_argument_with_invalid_apps(call_squash_migrations, monkeypatch): mock_squash = unittest.mock.MagicMock() - monkeypatch.setattr( - "django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", - mock_squash, - ) + monkeypatch.setattr("django_squash.db.migrations.autodetector.SquashMigrationAutodetector.squash", mock_squash) with pytest.raises(CommandError) as error: call_squash_migrations( "--only", From f499111e490529a4de2e9bd0d7db4a0a03e8f240 Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Tue, 6 Aug 2024 05:41:52 -0400 Subject: [PATCH 14/23] ...dunno --- .../restructure_migration_dependencies.py | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 django_squash/management/commands/restructure_migration_dependencies.py diff --git a/django_squash/management/commands/restructure_migration_dependencies.py b/django_squash/management/commands/restructure_migration_dependencies.py new file mode 100644 index 0000000..41bf39b --- /dev/null +++ b/django_squash/management/commands/restructure_migration_dependencies.py @@ -0,0 +1,133 @@ +import itertools +import os + +from django.apps import apps +from django.core.management.base import BaseCommand, CommandError, no_translations +from django.db.migrations.loader import MigrationLoader +from django.db.migrations.state import ProjectState + +from django_squash import settings as app_settings +from django_squash.db.migrations import serializer +from django_squash.db.migrations.autodetector import SquashMigrationAutodetector +from django_squash.db.migrations.loader import SquashMigrationLoader +from django_squash.db.migrations.questioner import NonInteractiveMigrationQuestioner +from django_squash.db.migrations.writer import MigrationWriter + + +class Command(BaseCommand): + def add_arguments(self, parser): + pass + + @no_translations + def handle(self, **kwargs): + self.verbosity = 1 + self.include_header = False + self.dry_run = kwargs["dry_run"] + + ignore_apps = [] + bad_apps = [] + + for app_label in kwargs["ignore_app"]: + try: + apps.get_app_config(app_label) + ignore_apps.append(app_label) + except (LookupError, TypeError): + bad_apps.append(str(app_label)) + + if kwargs["only"]: + only_apps = [] + + for app_label in kwargs["only"]: + try: + apps.get_app_config(app_label) + only_apps.append(app_label) + except (LookupError, TypeError): + bad_apps.append(app_label) + + for app_name in apps.app_configs.keys(): + if app_name not in only_apps: + ignore_apps.append(app_name) + + if bad_apps: + raise CommandError("The following apps are not valid: %s" % (", ".join(bad_apps))) + + questioner = NonInteractiveMigrationQuestioner(specified_apps=None, dry_run=False) + + loader = MigrationLoader(None, ignore_no_migrations=True) + squash_loader = SquashMigrationLoader(None, ignore_no_migrations=True) + + # Set up autodetector + autodetector = SquashMigrationAutodetector( + squash_loader.project_state(), + ProjectState.from_apps(apps), + questioner, + ) + + squashed_changes = autodetector.squash( + real_loader=loader, + squash_loader=squash_loader, + ignore_apps=ignore_apps, + migration_name=kwargs["squashed_name"], + ) + + replacing_migrations = 0 + for migration in itertools.chain.from_iterable(squashed_changes.values()): + replacing_migrations += len(migration.replaces) + + if not replacing_migrations: + raise CommandError("There are no migrations to squash.") + + self.write_migration_files(squashed_changes) + + @serializer.patch_serializer_registry + def write_migration_files(self, changes): + """ + Take a changes dict and write them out as migration files. + """ + directory_created = {} + for app_label, app_migrations in changes.items(): + if self.verbosity >= 1: + self.stdout.write(self.style.MIGRATE_HEADING("Migrations for '%s':" % app_label) + "\n") + for migration in app_migrations: + # Describe the migration + writer = MigrationWriter(migration, self.include_header) + if self.verbosity >= 1: + # Display a relative path if it's below the current working + # directory, or an absolute path otherwise. + try: + migration_string = os.path.relpath(writer.path) + except ValueError: + migration_string = writer.path + if migration_string.startswith(".."): + migration_string = writer.path + self.stdout.write(" %s\n" % (self.style.MIGRATE_LABEL(migration_string),)) + if hasattr(migration, "is_migration_level") and migration.is_migration_level: + for operation in migration.describe(): + self.stdout.write(" - %s\n" % operation) + else: + for operation in migration.operations: + self.stdout.write(" - %s\n" % operation.describe()) + if not self.dry_run: + # Write the migrations file to the disk. + migrations_directory = os.path.dirname(writer.path) + if not directory_created.get(app_label): + os.makedirs(migrations_directory, exist_ok=True) + init_path = os.path.join(migrations_directory, "__init__.py") + if not os.path.isfile(init_path): + open(init_path, "w").close() + # We just do this once per app + directory_created[app_label] = True + migration_string = writer.as_string() + if migration_string is None: + # File was deleted + continue + with open(writer.path, "w", encoding="utf-8") as fh: + fh.write(migration_string) + elif self.verbosity == 3: + # Alternatively, makemigrations --dry-run --verbosity 3 + # will output the migrations to stdout rather than saving + # the file to the disk. + self.stdout.write( + self.style.MIGRATE_HEADING("Full migrations file '%s':" % writer.filename) + "\n" + ) + self.stdout.write("%s\n" % writer.as_string()) From c11ede28ecf7c91787a1fc3e22a9db8a7f557b06 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Mon, 25 Nov 2024 16:17:20 -0500 Subject: [PATCH 15/23] Update circularmigrations.py --- django_squash/management/commands/circularmigrations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index 0483825..ed3f81b 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -81,3 +81,5 @@ def handle(self, **kwargs): print( f" references {model._meta.app_label}.{model._meta.object_name} via {model_name}.{field_name}" ) + if references_found: + raise CommandError("Bad dependencies found") From 7167ee584d41c743c4012a7c4c2caca4b7e65fd4 Mon Sep 17 00:00:00 2001 From: kingbuzzman Date: Wed, 26 Mar 2025 10:12:22 -0400 Subject: [PATCH 16/23] wip --- django_squash/db/migrations/autodetector.py | 16 ++-- django_squash/db/migrations/loader.py | 34 ++++---- django_squash/db/migrations/utils.py | 78 +++++++------------ django_squash/db/migrations/writer.py | 20 +---- .../management/commands/squash_migrations.py | 1 + django_squash/settings.py | 1 - 6 files changed, 51 insertions(+), 99 deletions(-) diff --git a/django_squash/db/migrations/autodetector.py b/django_squash/db/migrations/autodetector.py index 9b942b0..192c8c1 100644 --- a/django_squash/db/migrations/autodetector.py +++ b/django_squash/db/migrations/autodetector.py @@ -10,7 +10,8 @@ from django.db.migrations.autodetector import MigrationAutodetector as MigrationAutodetectorBase from django_squash.contrib import postgres -from django_squash.db.migrations import utils + +from . import utils RESERVED_MIGRATION_KEYWORDS = ("_deleted", "_dependencies_change", "_replaces_change", "_original_migration") @@ -83,13 +84,10 @@ def add_non_elidables(self, loader, changes): continue if isinstance(operation, dj_migrations.RunSQL): - operation._original_migration = migration new_operations.append(operation) elif isinstance(operation, dj_migrations.RunPython): - operation._original_migration = migration new_operations.append(operation) elif isinstance(operation, postgres.PGCreateExtension): - operation._original_migration = migration new_operations_bubble_top.append(operation) elif isinstance(operation, dj_migrations.SeparateDatabaseAndState): # A valid use case for this should be given before any work is done. @@ -160,7 +158,7 @@ def convert_migration_references_to_objects(self, original, changes, ignore_apps for dependency in migration.dependencies: dep_app_label, dep_migration = dependency if dep_app_label in ignore_apps: - new_dependencies.append(original.graph.leaf_nodes(dep_app_label)[0]) + new_dependencies.append(original.graph.leaf_nodes(dependency[0])[0]) continue if dep_app_label == "__setting__": @@ -174,10 +172,10 @@ def convert_migration_references_to_objects(self, original, changes, ignore_apps # Leave as is, the django's migration writer will handle this by default new_dependencies.append(dependency) continue - # Technically, the terms '__first__' and '__latest__' could apply to dependencies. However, these - # are not labels that Django assigns automatically. Instead, they would be manually specified by - # the developer after Django has generated the necessary files. Given that our focus is solely - # on handling migrations created by Django, there is no practical need to account for these. + elif dep_migration == "__first__": + dependency = original.graph.root_nodes(dep_app_label)[0] + elif dep_migration == "__latest__": + dependency = original.graph.leaf_nodes(dep_app_label)[0] migration_id = dependency if migration_id not in migrations_by_name: diff --git a/django_squash/db/migrations/loader.py b/django_squash/db/migrations/loader.py index 257989e..1c76f54 100644 --- a/django_squash/db/migrations/loader.py +++ b/django_squash/db/migrations/loader.py @@ -1,4 +1,4 @@ -import logging +import inspect import os import tempfile from contextlib import ExitStack @@ -7,10 +7,6 @@ from django.conf import settings from django.db.migrations.loader import MigrationLoader -from django_squash.db.migrations import utils - -logger = logging.getLogger(__name__) - class SquashMigrationLoader(MigrationLoader): def __init__(self, *args, **kwargs): @@ -18,27 +14,23 @@ def __init__(self, *args, **kwargs): original_migration_modules = settings.MIGRATION_MODULES # make a copy of the migration modules so we can modify it settings.MIGRATION_MODULES = settings.MIGRATION_MODULES.copy() - site_packages_path = utils.site_packages_path() + project_path = os.path.abspath(os.curdir) with ExitStack() as stack: # Find each app that belongs to the user and are not in the site-packages. Create a fake temporary # directory inside each app that will tell django we don't have any migrations at all. for app_config in apps.get_app_configs(): - # absolute path to the app - app_path = utils.source_directory(app_config.module) - - if app_path.startswith(site_packages_path): - # ignore any apps in inside site-packages - logger.debug("Ignoring app %s inside site-packages: %s", app_config.label, app_path) - continue - - temp_dir = stack.enter_context(tempfile.TemporaryDirectory(prefix="migrations_", dir=app_path)) - # Need to make this directory a proper python module otherwise django will refuse to recognize it - open(os.path.join(temp_dir, "__init__.py"), "a").close() - settings.MIGRATION_MODULES[app_config.label] = "%s.%s" % ( - app_config.module.__name__, - os.path.basename(temp_dir), - ) + module = app_config.module + app_path = os.path.dirname(os.path.abspath(inspect.getsourcefile(module))) + + if app_path.startswith(project_path): + temp_dir = stack.enter_context(tempfile.TemporaryDirectory(prefix="migrations_", dir=app_path)) + # Need to make this directory a proper python module otherwise django will refuse to recognize it + open(os.path.join(temp_dir, "__init__.py"), "a").close() + settings.MIGRATION_MODULES[app_config.label] = "%s.%s" % ( + app_config.module.__name__, + os.path.basename(temp_dir), + ) super().__init__(*args, **kwargs) diff --git a/django_squash/db/migrations/utils.py b/django_squash/db/migrations/utils.py index 1279c00..2abe985 100644 --- a/django_squash/db/migrations/utils.py +++ b/django_squash/db/migrations/utils.py @@ -1,5 +1,4 @@ import ast -import functools import hashlib import importlib import inspect @@ -13,19 +12,6 @@ from django.db import migrations from django.utils.module_loading import import_string -from django_squash import settings as app_settings - - -@functools.lru_cache(maxsize=1) -def get_custom_rename_function(): - """ - Custom function naming when copying elidable functions from one file to another. - """ - custom_rename_function = app_settings.DJANGO_SQUASH_CUSTOM_RENAME_FUNCTION - - if custom_rename_function: - return import_string(custom_rename_function) - def file_hash(file_path): """ @@ -44,9 +30,6 @@ def file_hash(file_path): def source_directory(module): - """ - Return the absolute path of a module - """ return os.path.dirname(os.path.abspath(inspect.getsourcefile(module))) @@ -55,14 +38,9 @@ class UniqueVariableName: This class will return a unique name for a variable / function. """ - def __init__(self, context, naming_function=None): + def __init__(self): self.names = defaultdict(int) self.functions = {} - self.context = context - self.naming_function = naming_function or (lambda n, c: n) - - def update_context(self, context): - self.context.update(context) def function(self, func): if not callable(func): @@ -74,37 +52,39 @@ def function(self, func): if inspect.ismethod(func) or inspect.signature(func).parameters.get("self") is not None: raise ValueError("func cannot be part of an instance") - name = func.__qualname__ + name = original_name = func.__qualname__ if "." in name: parent_name, actual_name = name.rsplit(".", 1) parent = getattr(import_string(func.__module__), parent_name) if issubclass(parent, migrations.Migration): - name = actual_name - - if func in self.functions: + name = name = original_name = actual_name + already_accounted = func in self.functions + if already_accounted: return self.functions[func] - name = self.naming_function(name, {**self.context, "type_": "function", "func": func}) - new_name = self.functions[func] = self.uniq(name) - - return new_name - - def uniq(self, name, original_name=None): - original_name = original_name or name # Endless loop that will try different combinations until it finds a unique name for i, _ in enumerate(itertools.count(), 2): if self.names[name] == 0: + self.functions[func] = name self.names[name] += 1 break name = "%s_%s" % (original_name, i) + + self.functions[func] = name + return name def __call__(self, name, force_number=False): - original_name = name - if force_number: - name = f"{name}_1" - return self.uniq(name, original_name) + self.names[name] += 1 + count = self.names[name] + if not force_number and count == 1: + return name + else: + new_name = "%s_%s" % (name, count) + # Make sure that the function name is fully unique + # You can potentially have the same name already defined. + return self(new_name) def get_imports(module): @@ -129,10 +109,9 @@ def get_imports(module): def normalize_function_name(name): - _, _, function_name = name.rpartition(".") - if function_name[0].isdigit(): - # Functions CANNOT start with a number - function_name = "f_" + function_name + class_name, _, function_name = name.rpartition(".") + if class_name and not function_name: + function_name = class_name return function_name @@ -144,7 +123,10 @@ def copy_func(f, name): func.__qualname__ = name func.__original__ = f func.__source__ = re.sub( - rf"(def\s+){normalize_function_name(f.__qualname__)}", rf"\1{name}", inspect.getsource(f), 1 + rf"(def\s+){normalize_function_name(f.__qualname__)}", + rf"\1{normalize_function_name(name)}", + inspect.getsource(f), + 1, ) return func @@ -164,18 +146,12 @@ def find_brackets(line, p_count, b_count): def is_code_in_site_packages(module_name): # Find the module in the site-packages directory - site_packages_path_ = site_packages_path() + site_packages_path = sysconfig.get_path("purelib") # returns the "../site-packages" directory try: loader = importlib.util.find_spec(module_name) + return site_packages_path in loader.origin except ImportError: return False - return loader.origin.startswith(site_packages_path_) - - -@functools.lru_cache(maxsize=1) -def site_packages_path(): - # returns the "../site-packages" directory - return sysconfig.get_path("purelib") def replace_migration_attribute(source, attr, value): diff --git a/django_squash/db/migrations/writer.py b/django_squash/db/migrations/writer.py index 5645983..e64a284 100644 --- a/django_squash/db/migrations/writer.py +++ b/django_squash/db/migrations/writer.py @@ -166,20 +166,8 @@ def as_string(self): return self.replace_in_migration() variables = [] - custom_naming_function = utils.get_custom_rename_function() - unique_names = utils.UniqueVariableName( - {"app": self.migration.app_label}, naming_function=custom_naming_function - ) + unique_names = utils.UniqueVariableName() for operation in self.migration.operations: - unique_names.update_context( - { - "new_migration": self.migration, - "operation": operation, - "migration": ( - operation._original_migration if hasattr(operation, "_original_migration") else self.migration - ), - } - ) operation._deconstruct = operation.__class__.deconstruct def deconstruct(self): @@ -191,14 +179,12 @@ def deconstruct(self): # Bind the deconstruct() to the instance to get the elidable operation.deconstruct = deconstruct.__get__(operation, operation.__class__) if not utils.is_code_in_site_packages(operation.code.__module__): - code_name = utils.normalize_function_name(unique_names.function(operation.code)) + code_name = unique_names.function(operation.code) operation.code = utils.copy_func(operation.code, code_name) operation.code.__in_migration_file__ = True if operation.reverse_code: if not utils.is_code_in_site_packages(operation.reverse_code.__module__): - reversed_code_name = utils.normalize_function_name( - unique_names.function(operation.reverse_code) - ) + reversed_code_name = unique_names.function(operation.reverse_code) operation.reverse_code = utils.copy_func(operation.reverse_code, reversed_code_name) operation.reverse_code.__in_migration_file__ = True elif isinstance(operation, dj_migrations.RunSQL): diff --git a/django_squash/management/commands/squash_migrations.py b/django_squash/management/commands/squash_migrations.py index 46fc739..d745b14 100644 --- a/django_squash/management/commands/squash_migrations.py +++ b/django_squash/management/commands/squash_migrations.py @@ -60,6 +60,7 @@ def handle(self, **kwargs): try: apps.get_app_config(app_label) only_apps.append(app_label) + # Edge case: if the app was previously ignored, remove it from the ignore list if app_label in ignore_apps: raise CommandError( "The following app cannot be ignored and selected at the same time: %s" % app_label diff --git a/django_squash/settings.py b/django_squash/settings.py index d33911b..505aa75 100644 --- a/django_squash/settings.py +++ b/django_squash/settings.py @@ -2,4 +2,3 @@ DJANGO_SQUASH_IGNORE_APPS = getattr(global_settings, "DJANGO_SQUASH_IGNORE_APPS", None) or [] DJANGO_SQUASH_MIGRATION_NAME = getattr(global_settings, "DJANGO_SQUASH_MIGRATION_NAME", None) or "squashed" -DJANGO_SQUASH_CUSTOM_RENAME_FUNCTION = getattr(global_settings, "DJANGO_SQUASH_CUSTOM_RENAME_FUNCTION", None) From dda05c1703b66bf9b00719ded6463e93b1a4a84e Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Wed, 30 Jul 2025 11:23:32 +0200 Subject: [PATCH 17/23] Gettings missing test coverage always --- .github/workflows/_tests.yml | 10 ++++++++++ pyproject.toml | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index e759f3a..c74c397 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -166,6 +166,8 @@ jobs: - name: Install packages run: uv pip install --system -e '.[test]' - name: Run tests + id: pytest + continue-on-error: true run: pytest --cov . - name: Upload coverage artifact uses: actions/upload-artifact@v4 @@ -175,6 +177,14 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: .coverage retention-days: 1 + - name: Validate run + run: | + if [ "${STEPS_PYTEST_OUTCOME}" != "success" ]; then + echo "Tests failed!" + exit 1 + fi + env: + STEPS_PYTEST_OUTCOME: ${{ steps.pytest.outcome }} coverage: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index fd017c2..9578d69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,7 +152,6 @@ omit = ["*/migrations_*/*"] [tool.coverage.report] show_missing = true -fail_under = 90 [tool.coverage.html] show_contexts = true From 94def18aa88531100d85fce44dbb142308d875e6 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Wed, 30 Jul 2025 11:47:56 +0200 Subject: [PATCH 18/23] Always the diff --- .github/workflows/_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index c74c397..438743f 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -214,7 +214,6 @@ jobs: coverage combine $(find downloaded_artifacts/ -type f | xargs) # Used by codecov coverage xml - coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - name: Upload single coverage artifact uses: actions/upload-artifact@v4 with: @@ -225,7 +224,6 @@ jobs: retention-days: 1 - name: Diff coverage run: | - coverage xml diff-cover coverage.xml --compare-branch origin/master --format github-annotations:warning - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 @@ -234,3 +232,5 @@ jobs: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} env_vars: OS,RUST + - name: Check + run: coverage report --format=markdown --fail-under=90 >> $GITHUB_STEP_SUMMARY From e023128c70020f59bce48f3e3a645eeefc4eab84 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Wed, 30 Jul 2025 11:56:59 +0200 Subject: [PATCH 19/23] Fix import --- django_squash/db/migrations/utils.py | 1 + .../management/commands/restructure_migration_dependencies.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/django_squash/db/migrations/utils.py b/django_squash/db/migrations/utils.py index 4fe7187..7cac1f8 100644 --- a/django_squash/db/migrations/utils.py +++ b/django_squash/db/migrations/utils.py @@ -8,6 +8,7 @@ import sysconfig import types from collections import defaultdict +import functools from django.db import migrations from django.utils.module_loading import import_string diff --git a/django_squash/management/commands/restructure_migration_dependencies.py b/django_squash/management/commands/restructure_migration_dependencies.py index 41bf39b..40b57ba 100644 --- a/django_squash/management/commands/restructure_migration_dependencies.py +++ b/django_squash/management/commands/restructure_migration_dependencies.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import itertools import os @@ -6,7 +8,6 @@ from django.db.migrations.loader import MigrationLoader from django.db.migrations.state import ProjectState -from django_squash import settings as app_settings from django_squash.db.migrations import serializer from django_squash.db.migrations.autodetector import SquashMigrationAutodetector from django_squash.db.migrations.loader import SquashMigrationLoader From 042f3a444895d4fd88e346f8f1bfae124bf6b7e3 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Wed, 30 Jul 2025 12:53:00 +0200 Subject: [PATCH 20/23] . --- django_squash/management/commands/circularmigrations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index ed3f81b..8571b01 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -58,7 +58,8 @@ def handle(self, **kwargs): references_found = [] for operation in migration.operations: if hasattr(operation, "field") and hasattr(operation.field, "related_model"): - if operation.field.related_model._meta.app_label in bad_dependencies: + if apps.get_model(operation.field.related_model)._meta.app_label in bad_dependencies: + # if operation.field.related_model._meta.app_label in bad_dependencies: references_found.append( ( operation.model_name, @@ -74,7 +75,8 @@ def handle(self, **kwargs): and field.related_model and not isinstance(field.related_model, str) ): - if field.related_model._meta.app_label in bad_dependencies: + if apps.get_model(operation.field.related_model)._meta.app_label in bad_dependencies: + # if field.related_model._meta.app_label in bad_dependencies: references_found.append((operation.name, name, field.related_model)) for model_name, field_name, model in references_found: From c1ee24ead75a6f86aff751ab23676d2d32800253 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Wed, 30 Jul 2025 13:05:18 +0200 Subject: [PATCH 21/23] . --- .../management/commands/circularmigrations.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index 8571b01..b9679ec 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -58,8 +58,7 @@ def handle(self, **kwargs): references_found = [] for operation in migration.operations: if hasattr(operation, "field") and hasattr(operation.field, "related_model"): - if apps.get_model(operation.field.related_model)._meta.app_label in bad_dependencies: - # if operation.field.related_model._meta.app_label in bad_dependencies: + if get_related_model(operation.field.related_model)._meta.app_label in bad_dependencies: references_found.append( ( operation.model_name, @@ -68,16 +67,10 @@ def handle(self, **kwargs): ) ) - if hasattr(operation, "fields"): - for name, field in operation.fields: - if ( - hasattr(field, "related_model") - and field.related_model - and not isinstance(field.related_model, str) - ): - if apps.get_model(operation.field.related_model)._meta.app_label in bad_dependencies: - # if field.related_model._meta.app_label in bad_dependencies: - references_found.append((operation.name, name, field.related_model)) + for name, field in getattr(operation, "fields", []): + if (hasattr(field, "related_model")): + if get_related_model(field.related_model)._meta.app_label in bad_dependencies: + references_found.append((operation.name, name, get_related_model(field.related_model))) for model_name, field_name, model in references_found: print( @@ -85,3 +78,9 @@ def handle(self, **kwargs): ) if references_found: raise CommandError("Bad dependencies found") + + +def get_related_model(model): + if isinstance(model, str): + return apps.get_model(model) + return model \ No newline at end of file From a647cd7c47c0b141e5c0f64724d61329ab3b44a6 Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Wed, 30 Jul 2025 13:07:17 +0200 Subject: [PATCH 22/23] . --- django_squash/management/commands/circularmigrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index b9679ec..83ea841 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -63,7 +63,7 @@ def handle(self, **kwargs): ( operation.model_name, operation.name, - operation.field.related_model, + get_related_model(operation.field.related_model), ) ) From fc9cef7886c2af39faa09dba2977747b0337243b Mon Sep 17 00:00:00 2001 From: Javier Buzzi Date: Wed, 30 Jul 2025 13:27:40 +0200 Subject: [PATCH 23/23] . --- django_squash/management/commands/circularmigrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_squash/management/commands/circularmigrations.py b/django_squash/management/commands/circularmigrations.py index 83ea841..8518068 100644 --- a/django_squash/management/commands/circularmigrations.py +++ b/django_squash/management/commands/circularmigrations.py @@ -1,6 +1,6 @@ from django.apps import apps from django.conf import settings -from django.core.management.base import BaseCommand, no_translations +from django.core.management.base import BaseCommand, no_translations, CommandError from django.db.migrations.state import ProjectState from django_squash.db.migrations.autodetector import SquashMigrationAutodetector