New migration library

.github/workflows/golangci-lint.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-go@v6
         with:
-          go-version: '1.21'
+          go-version: '1.24'
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v9

.github/workflows/run-tests.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: '1.24'
 
       - name: Debug environment variables
         run: printenv | sort
@@ -67,7 +67,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.21
+          go-version: '1.24'
 
       - name: Install dependencies
         run: go mod tidy

CLAUDE.md

@@ -62,6 +62,6 @@ Handler → Service → Repository → DB
 
 **Request/Response** (`internal/request/`) — `GetRequestParams` parses pagination, filtering, and sorting from query params. Filters are declared per-handler as `[]req.FilterRule`; sortable fields are whitelisted. Response helpers (`GetSingleItemResp`, `GetListResp`, `GetValidateErrResp`, etc.) standardise the JSON envelope.
 
-**Migrations** (`migrations/`) — raw SQL files named `<timestamp>_<name>.up.sql` / `.down.sql`. Multiple statements are separated by `------------------` on its own line. Tracking is via a `migrations` table in the DB.
+**Migrations** (`migrations/`) — raw SQL files named `<timestamp>_<name>.up.sql` / `.down.sql` where timestamp is `YYYYMMDDHHmmss` (14 digits, no separators). Multiple statements per file are separated by `;`. Tracking is via a `schema_migrations` table managed by [golang-migrate/migrate](https://github.com/golang-migrate/migrate). If a migration fails mid-run the DB is marked dirty; resolve with `migrate force <version>` before retrying.
 
 **Scaffolding** (`internal/scaffold/`, `cmd/scaffold/`) — reads a live DB table schema and generates the four layers above. Output paths are configured in `cmd/scaffold/config.json`.

Makefile

@@ -15,7 +15,7 @@ lint:
 
 .PHONY: test
 test:
-	go test ./... -cover
+	go test -p 1 ./... -cover
 
 SHELL := /bin/bash
 docker_up:

README.md

@@ -14,8 +14,8 @@ The basic requirements from a RESTful API framework are:
 - ~~easy filtering in collection requests~~
 - ~~sorting and ordering in collection requests~~
 - ~~CRUD scaffolding~~
-- user model, login and authentication, etc...
-- ci/cd
+- ~~user model, login and authentication, etc...~~
+- ~~ci/cd~~
 - documentation
 
 Good to have:
@@ -29,12 +29,43 @@ Good to have:
 - `./internal/models/` # Contain data models
 - `./internal/service/` # Service layer that contains business logic of each model/endpoints
 - `./internal/repository/` # Contains data retrieval functions
-- `./internal/migrations/` # Contains database migration files
+- `./migrations/` # Contains database migration files
 - `./internal/middleware/` # Contains API middleware
 - `./config.json` # Application configuration
 
-## Migration
-Migration will run SQL commands to update the database. It has its own [README](README_MIGRATION.md).
+## Migrations
+
+Migration files live in `./migrations/` and are named `<timestamp>_<name>.up.sql` / `<timestamp>_<name>.down.sql` where the timestamp is 14 digits (`YYYYMMDDHHmmss`). Multiple SQL statements in one file are separated by `;`. Run state is tracked in a `schema_migrations` table managed by [golang-migrate/migrate](https://github.com/golang-migrate/migrate).
+
+**Create a new migration:**
+```bash
+go run ./cmd/migrate/migrate.go -action create -name <migration_name>
+```
+
+**Run all pending migrations:**
+```bash
+go run ./cmd/migrate/migrate.go -action up
+```
+
+**Run a specific number of migrations:**
+```bash
+go run ./cmd/migrate/migrate.go -action up -number 1
+```
+
+**Rollback one migration:**
+```bash
+go run ./cmd/migrate/migrate.go -action down
+```
+
+**Rollback a specific number of migrations:**
+```bash
+go run ./cmd/migrate/migrate.go -action down -number 2
+```
+
+If a migration fails mid-run the database is marked dirty. Resolve with:
+```bash
+migrate force <version>
+```
 
 ## Scaffolding
 Scaffolding is a tool that will read the table definition in the database and create the CRUD routes, handlers, service and

ci/.golangci-lint.yml

@@ -58,6 +58,11 @@ linters:
     gosec:
       excludes:
         - G101
+  exclusions:
+    rules:
+      - path: "_test.go"
+        linters:
+          - goconst
 
 formatters:
   enable:

cmd/migrate/migrate.go

@@ -2,17 +2,19 @@
 package main
 
 import (
+	"database/sql"
 	"flag"
 	"fmt"
 	"log"
 	"os"
+	"time"
 
 	_ "github.com/go-sql-driver/mysql"
+	gm "github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/mysql"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
 
 	"github.com/doublehops/dh-go-framework/internal/config"
-	"github.com/doublehops/dh-go-framework/internal/db"
-	"github.com/doublehops/dh-go-framework/internal/logga"
-	migrate "github.com/doublehops/dh-go-framework/internal/migration"
 )
 
 func main() {
@@ -23,68 +25,121 @@ func main() {
 }
 
 func run() error {
-	var args migrate.Action
+	action := flag.String("action", "", "action: up, down, create")
+	name := flag.String("name", "", "migration name (required for create)")
+	number := flag.Int("number", 0, "number of migrations to run (0 = all for up, 1 for down)")
+	configFile := flag.String("config", "config.json", "config file")
+	flag.Parse()
 
-	flag.StringVar(&args.Action, "action", "", "the intended action")
-	flag.StringVar(&args.Name, "name", "", "the name of the migration")
-	flag.IntVar(&args.Number, "number", 0, "The number of migrations to run")
+	if *action == "" {
+		fmt.Fprintln(os.Stderr, "Usage: -action [up|down|create] [-number N] [-name <name>]")
+		os.Exit(1)
+	}
 
-	configFile := flag.String("config", "config.json", "Config file to use")
-	flag.Parse()
+	if *action == "create" {
+		if *name == "" {
+			fmt.Fprintln(os.Stderr, "-name is required for the create action")
+			os.Exit(1)
+		}
 
-	args = setFlags(args)
+		return createMigration(*name)
+	}
 
-	// Setup config.
 	cfg, err := config.New(*configFile)
 	if err != nil {
-		return fmt.Errorf("error starting main. %s", err.Error())
+		return fmt.Errorf("error loading config: %w", err)
 	}
 
-	// Setup logger.
-	l, err := logga.New(&cfg.Logging)
+	m, cleanup, err := newMigrator(cfg)
 	if err != nil {
-		return fmt.Errorf("error configuring logger. %s", err.Error())
+		return err
+	}
+
+	defer cleanup()
+
+	return runAction(m, *action, *number)
+}
+
+func runAction(m *gm.Migrate, action string, number int) error {
+	version, dirty, _ := m.Version() //nolint:errcheck
+	log.Printf("current version: %d (dirty: %v)", version, dirty)
+
+	var err error
+
+	switch action {
+	case "up":
+		if number == 0 {
+			err = m.Up()
+		} else {
+			err = m.Steps(number)
+		}
+	case "down":
+		n := number
+		if n == 0 {
+			n = 1
+		}
+
+		err = m.Steps(-n)
+	default:
+		return fmt.Errorf("unknown action %q — use up, down, or create", action)
 	}
 
-	// Setup db connection.
-	DB, err := db.New(l, cfg.DB)
+	if err == gm.ErrNoChange {
+		log.Println("no migrations to run")
+
+		return nil
+	}
+
+	return err
+}
+
+func newMigrator(cfg *config.Config) (*gm.Migrate, func(), error) {
+	dsn := fmt.Sprintf("%s:%s@(%s:3306)/%s?parseTime=true&multiStatements=true",
+		cfg.DB.User, cfg.DB.Pass, cfg.DB.Host, cfg.DB.Name)
+
+	sqlDB, err := sql.Open("mysql", dsn)
 	if err != nil {
-		return fmt.Errorf("error creating database connection. %s", err.Error())
+		return nil, nil, fmt.Errorf("error opening database: %w", err)
 	}
 
-	dir, err := os.Getwd()
+	driver, err := mysql.WithInstance(sqlDB, &mysql.Config{})
 	if err != nil {
-		return fmt.Errorf("there was an error with os.Getwd(). %s", err.Error())
+		_ = sqlDB.Close()
+		return nil, nil, fmt.Errorf("error creating mysql driver: %w", err)
 	}
 
-	args.Path = dir + "/migrations"
-	args.DB = DB
+	dir, err := os.Getwd()
+	if err != nil {
+		_ = sqlDB.Close()
+		return nil, nil, fmt.Errorf("error getting working directory: %w", err)
+	}
 
-	err = args.Migrate()
+	m, err := gm.NewWithDatabaseInstance("file://"+dir+"/migrations", "mysql", driver)
 	if err != nil {
-		return fmt.Errorf("there was an error initialising migration. %s", err.Error())
+		_ = sqlDB.Close()
+		return nil, nil, fmt.Errorf("error creating migrator: %w", err)
 	}
 
-	return nil
+	return m, func() { _ = sqlDB.Close() }, nil
 }
 
-// setFlags will check that the flags received are valid and assign default ones if not supplied.
-func setFlags(args migrate.Action) migrate.Action {
-	if found := args.IsValidAction(args.Action); !found {
-		args.PrintHelp()
+func createMigration(name string) error {
+	dir, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("error getting working directory: %w", err)
 	}
 
-	if args.Action == "create" && args.Name == "" {
-		args.PrintHelp()
-	}
+	ts := time.Now().Format("20060102150405")
+	base := fmt.Sprintf("%s/migrations/%s_%s", dir, ts, name)
 
-	if args.Action == "up" && args.Number == 0 {
-		args.Number = 9999 // run them all if none defined.
+	if err := os.WriteFile(base+".up.sql", []byte("-- Write your up migration SQL here\n"), 0o600); err != nil {
+		return fmt.Errorf("error creating up migration file: %w", err)
 	}
-
-	if args.Action == "down" && args.Number == 0 {
-		args.Number = 1 // run just one if none defined.
+	if err := os.WriteFile(base+".down.sql", []byte("-- Write your rollback SQL here\n"), 0o600); err != nil {
+		return fmt.Errorf("error creating down migration file: %w", err)
 	}
 
-	return args
+	log.Printf("created:\n  %s.up.sql\n  %s.down.sql", base, base)
+
+	return nil
 }

go.mod

@@ -1,27 +1,27 @@
 module github.com/doublehops/dh-go-framework
 
-go 1.21
+go 1.24.0
 
 // @todo - this should be removed when dhapi is pushed to Github.
 replace github.com/doublehops/dhapi => /home/b/workspace/dhapi-2
 
 require (
 	github.com/doublehops/go-common v0.0.0-20230910011642-8556bd635e3f
-	github.com/doublehops/go-migration v0.0.3
 	github.com/go-sql-driver/mysql v1.8.1
+	github.com/golang-migrate/migrate/v4 v4.19.1
 	github.com/jinzhu/copier v0.4.0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/joho/godotenv v1.5.1
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/mythrnr/httprouter-group v0.9.1
-	github.com/stretchr/testify v1.9.0
-	golang.org/x/crypto v0.27.0
-	golang.org/x/text v0.18.0
+	github.com/stretchr/testify v1.10.0
+	golang.org/x/crypto v0.45.0
+	golang.org/x/text v0.31.0
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-)
+)