Better migrations DX (#50)

* update migrations mechanism

* update docs

Author: mishankov

.agents/architecture.md

@@ -85,27 +85,33 @@ server.Handle("/api", api)
 
 ## DATABASE MIGRATIONS
 
-Migrations live in the repository and run automatically on startup:
+Migrations are written in `.sql` files and embedded via `fs.FS`. Each file contains up and down scripts separated by markers:
+
+```sql
+-- 001_create_users.sql
+-- +migrate Up
+CREATE TABLE users (
+    id UUID PRIMARY KEY,
+    email TEXT UNIQUE NOT NULL
+);
+
+-- +migrate Down
+DROP TABLE users;
+```
+
+Repository returns embedded migrations:
 
 ```go
-func (r *Repository) Migrations() []database.Migration {
-    return []database.Migration{
-        {
-            Name: "001_create_users_table",
-            Up: `CREATE TABLE users (
-                id UUID PRIMARY KEY,
-                email TEXT UNIQUE NOT NULL,
-                created_at TIMESTAMP NOT NULL DEFAULT NOW()
-            )`,
-        },
-        {
-            Name: "002_add_status_column",
-            Up: `ALTER TABLE users ADD COLUMN status TEXT NOT NULL DEFAULT 'active'`,
-        },
-    }
+//go:embed *.sql
+var migrations embed.FS
+
+func (r *Repository) Migrations() fs.FS {
+    return migrations
 }
 ```
 
+Migration files are sorted lexicographically by filename. Use numeric prefixes for ordering: `001_`, `002_`, etc.
+
 ## BACKGROUND JOBS
 
 Implement the queue handler interface:

auth/001_init.sql

@@ -0,0 +1,13 @@
+-- +migrate Up
+CREATE TABLE IF NOT EXISTS users (
+	id VARCHAR(255) PRIMARY KEY,
+	username VARCHAR(255) UNIQUE,
+	password TEXT,
+	salt TEXT,
+	created TIMESTAMP,
+	updated TIMESTAMP,
+	status VARCHAR(50)
+);
+
+-- +migrate Down
+DROP TABLE users;

auth/repository.go

@@ -3,9 +3,9 @@ package auth
 import (
 	"context"
 	"database/sql"
+	"embed"
 	"fmt"
-
-	"github.com/platforma-dev/platforma/database"
+	"io/fs"
 )
 
 type db interface {
@@ -25,21 +25,11 @@ func NewRepository(db db) *Repository {
 	}
 }
 
-func (r *Repository) Migrations() []database.Migration {
-	return []database.Migration{{
-		ID: "init",
-		Up: `CREATE TABLE IF NOT EXISTS users (
-			id VARCHAR(255) PRIMARY KEY,
-			username VARCHAR(255) UNIQUE,
-			password TEXT,
-			salt TEXT,
-			created TIMESTAMP,
-			updated TIMESTAMP,
-			status VARCHAR(50)
-		)`,
-		Down: "DROP TABLE users",
-	}}
+//go:embed *.sql
+var migrations embed.FS
 
+func (r *Repository) Migrations() fs.FS {
+	return migrations
 }
 
 func (r *Repository) Get(ctx context.Context, id string) (*User, error) {

database/database.go

@@ -61,7 +61,11 @@ func (db *Database) Migrate(ctx context.Context) error {
 	// Get migrations from all migrators
 	migrations := []Migration{}
 	for name, migrator := range db.migrators {
-		for _, migr := range migrator.Migrations() {
+		parsed, err := ParseMigrations(migrator.Migrations())
+		if err != nil {
+			return fmt.Errorf("failed to parse migrations for %s: %w", name, err)
+		}
+		for _, migr := range parsed {
 			migr.repository = name
 			migrations = append(migrations, migr)
 		}

database/database_test.go

@@ -4,8 +4,10 @@ package database_test
 
 import (
 	"context"
+	"io/fs"
 	"slices"
 	"testing"
+	"testing/fstest"
 	"time"
 
 	"github.com/platforma-dev/platforma/database"
@@ -148,11 +150,11 @@ func TestMigrate(t *testing.T) {
 			t.Fatalf("database is nil")
 		}
 
-		db.RegisterRepository("some_repo", simpleRepo{migrations: []database.Migration{{
-			ID:   "init",
+		db.RegisterRepository("some_repo", simpleRepo{fsys: migrationFS(database.Migration{
+			ID:   "001_init",
 			Up:   "CREATE TABLE IF NOT EXISTS simple_repo (id TEXT)",
 			Down: "DROP TABLE simple_repo",
-		}}})
+		})})
 
 		err = db.Migrate(ctx)
 		if err != nil {
@@ -171,7 +173,7 @@ func TestMigrate(t *testing.T) {
 		}
 
 		if !slices.ContainsFunc(migrationLogs, func(log migrationLog) bool {
-			return log.Repository == "some_repo" && log.MigrationID == "init"
+			return log.Repository == "some_repo" && log.MigrationID == "001_init"
 		}) {
 			t.Fatalf("expected migration log to contain init migration for some_repo")
 		}
@@ -199,17 +201,17 @@ func TestMigrate(t *testing.T) {
 			t.Fatalf("database is nil")
 		}
 
-		db.RegisterRepository("some_repo", simpleRepo{migrations: []database.Migration{{
-			ID:   "init",
+		db.RegisterRepository("some_repo", simpleRepo{fsys: migrationFS(database.Migration{
+			ID:   "001_init",
 			Up:   "CREATE TABLE IF NOT EXISTS simple_repo (id TEXT)",
 			Down: "DROP TABLE simple_repo",
-		}}})
+		})})
 
-		db.RegisterRepository("other_repo", simpleRepo{migrations: []database.Migration{{
-			ID:   "init",
+		db.RegisterRepository("other_repo", simpleRepo{fsys: migrationFS(database.Migration{
+			ID:   "001_init",
 			Up:   "CREATE TABLE IF NOT EXISTS other_repo (id TEXT)",
 			Down: "DROP TABLE other_repo",
-		}}})
+		})})
 
 		err = db.Migrate(ctx)
 		if err != nil {
@@ -228,7 +230,7 @@ func TestMigrate(t *testing.T) {
 		}
 
 		if !slices.ContainsFunc(migrationLogs, func(log migrationLog) bool {
-			return log.Repository == "some_repo" && log.MigrationID == "init"
+			return log.Repository == "some_repo" && log.MigrationID == "001_init"
 		}) {
 			t.Fatalf("expected migration log to contain init migration for some_repo")
 		}
@@ -239,7 +241,7 @@ func TestMigrate(t *testing.T) {
 		}
 
 		if !slices.ContainsFunc(migrationLogs, func(log migrationLog) bool {
-			return log.Repository == "other_repo" && log.MigrationID == "init"
+			return log.Repository == "other_repo" && log.MigrationID == "001_init"
 		}) {
 			t.Fatalf("expected migration log to contain init migration for other_repo, but only got: %s", migrationLogs)
 		}
@@ -267,21 +269,24 @@ func TestMigrate(t *testing.T) {
 			t.Fatalf("database is nil")
 		}
 
-		db.RegisterRepository("some_repo", simpleRepo{migrations: []database.Migration{{
-			ID:   "init",
+		db.RegisterRepository("some_repo", simpleRepo{fsys: migrationFS(database.Migration{
+			ID:   "001_init",
 			Up:   "CREATE TABLE IF NOT EXISTS simple_repo (id TEXT)",
 			Down: "DROP TABLE simple_repo",
-		}}})
-
-		db.RegisterRepository("other_repo", simpleRepo{migrations: []database.Migration{{
-			ID:   "init",
-			Up:   "CREATE TABLE IF NOT EXISTS other_repo (id TEXT)",
-			Down: "DROP TABLE other_repo",
-		}, {
-			ID:   "failing",
-			Up:   "not even SQL here",
-			Down: "no need for this",
-		}}})
+		})})
+
+		db.RegisterRepository("other_repo", simpleRepo{fsys: migrationFS(
+			database.Migration{
+				ID:   "001_init",
+				Up:   "CREATE TABLE IF NOT EXISTS other_repo (id TEXT)",
+				Down: "DROP TABLE other_repo",
+			},
+			database.Migration{
+				ID:   "002_failing",
+				Up:   "not even SQL here",
+				Down: "no need for this",
+			},
+		)})
 
 		err = db.Migrate(ctx)
 		if err == nil {
@@ -309,7 +314,7 @@ func TestMigrate(t *testing.T) {
 
 		// because migration should be reverted
 		if slices.ContainsFunc(migrationLogs, func(log migrationLog) bool {
-			return log.Repository == "some_repo" && log.MigrationID == "init"
+			return log.Repository == "some_repo" && log.MigrationID == "001_init"
 		}) {
 			t.Fatalf("expected migration log to not contain init migration for some_repo")
 		}
@@ -321,7 +326,7 @@ func TestMigrate(t *testing.T) {
 
 		// because migration should be reverted
 		if slices.ContainsFunc(migrationLogs, func(log migrationLog) bool {
-			return log.Repository == "other_repo" && log.MigrationID == "init"
+			return log.Repository == "other_repo" && log.MigrationID == "001_init"
 		}) {
 			t.Fatalf("expected migration log to not contain init migration for other_repo, but only got: %s", migrationLogs)
 		}
@@ -349,21 +354,24 @@ func TestMigrate(t *testing.T) {
 			t.Fatalf("database is nil")
 		}
 
-		db.RegisterRepository("some_repo", simpleRepo{migrations: []database.Migration{{
-			ID:   "init",
+		db.RegisterRepository("some_repo", simpleRepo{fsys: migrationFS(database.Migration{
+			ID:   "001_init",
 			Up:   "CREATE TABLE IF NOT EXISTS simple_repo (id TEXT)",
 			Down: "broken SQL",
-		}}})
-
-		db.RegisterRepository("other_repo", simpleRepo{migrations: []database.Migration{{
-			ID:   "init",
-			Up:   "CREATE TABLE IF NOT EXISTS other_repo (id TEXT)",
-			Down: "DROP TABLE other_repo",
-		}, {
-			ID:   "failing",
-			Up:   "not even SQL here",
-			Down: "no need for this",
-		}}})
+		})})
+
+		db.RegisterRepository("other_repo", simpleRepo{fsys: migrationFS(
+			database.Migration{
+				ID:   "001_init",
+				Up:   "CREATE TABLE IF NOT EXISTS other_repo (id TEXT)",
+				Down: "DROP TABLE other_repo",
+			},
+			database.Migration{
+				ID:   "002_failing",
+				Up:   "not even SQL here",
+				Down: "no need for this",
+			},
+		)})
 
 		err = db.Migrate(ctx)
 		if err == nil {
@@ -391,14 +399,14 @@ func TestMigrate(t *testing.T) {
 
 		// because migration should be reverted or not even attempted
 		if slices.ContainsFunc(migrationLogs, func(log migrationLog) bool {
-			return log.Repository == "some_repo" && log.MigrationID == "init"
+			return log.Repository == "some_repo" && log.MigrationID == "001_init"
 		}) {
 			t.Fatalf("expected migration log to not contain init migration for some_repo")
 		}
 
 		// because migration should be reverted
 		if slices.ContainsFunc(migrationLogs, func(log migrationLog) bool {
-			return log.Repository == "other_repo" && log.MigrationID == "init"
+			return log.Repository == "other_repo" && log.MigrationID == "001_init"
 		}) {
 			t.Fatalf("expected migration log to not contain init migration for other_repo, but only got: %s", migrationLogs)
 		}
@@ -417,9 +425,21 @@ type migrationLog struct {
 }
 
 type simpleRepo struct {
-	migrations []database.Migration
+	fsys fs.FS
+}
+
+func (r simpleRepo) Migrations() fs.FS {
+	return r.fsys
 }
 
-func (r simpleRepo) Migrations() []database.Migration {
-	return r.migrations
+func migrationFS(migrations ...database.Migration) fs.FS {
+	mapFS := make(fstest.MapFS)
+	for _, m := range migrations {
+		content := "-- +migrate Up\n" + m.Up
+		if m.Down != "" {
+			content += "\n\n-- +migrate Down\n" + m.Down
+		}
+		mapFS[m.ID+".sql"] = &fstest.MapFile{Data: []byte(content)}
+	}
+	return mapFS
 }

database/migration.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"io/fs"
 	"time"
 )
 
@@ -19,5 +20,5 @@ type Migration struct {
 }
 
 type migrator interface {
-	Migrations() []Migration
+	Migrations() fs.FS
 }