Merge pull request #50 from elcengine/refactor/populate

Refactor schema type validation and population usage

Author: Akalanka47000

.github/workflows/tests.yml

@@ -57,6 +57,8 @@ jobs:
           env:
             DEFAULT_DATASOURCE: mongodb://127.0.0.1:27017/elemental?replicaSet=rs0
             SECONDARY_DATASOURCE: mongodb://127.0.0.1:27018/elemental?replicaSet=rs1
+            GOEXPERIMENT: ${{ matrix.go-version == '1.23' && 'aliastypeparams' || '' }}
+            GODEBUG: ${{ matrix.go-version == '1.23' && 'gotypesalias=1' || '' }}
 
         - name: Upload coverage report
           if: github.event_name == 'pull_request' && github.base_ref == 'main' && matrix.go-version == '1.24' && matrix.mongo-version == '8.0'

.gitignore

@@ -6,4 +6,6 @@ database
 
 .elemental
 
-.vscode
+.vscode
+
+.env

Makefile

@@ -1,12 +1,11 @@
-GO_TEST_ARGS ?= -tags=unit
 GO_BENCH_ARGS ?= -benchtime=30s
 
 format:
 	gofmt -w .
 test:
 	PARALLEL_CONVEY=false make test-lightspeed
 test-lightspeed:
-	go test $(GO_TEST_ARGS) -v --count=1 ./tests/...
+	go test $(GO_TEST_ARGS) --run='${run}' -v --count=1 ./tests/...
 test-coverage:
 	@mkdir -p ./coverage
 	make test GO_TEST_ARGS="--cover -coverpkg=./cmd/...,./core/...,./plugins/...,./utils/... --coverprofile=./coverage/coverage.out"
@@ -30,9 +29,16 @@ install:
 	go install github.com/evilmartians/lefthook@v1.11.12
 	lefthook install
 	@echo "\033[0;32mLefthook installed and configured successfully.\033[0m"
+	go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.1.6
+	@echo "\033[0;32mGolangCI-Lint installed successfully.\033[0m"
 	@which npm > /dev/null && \
 		npm install -g @commitlint/config-conventional@17.6.5 @commitlint/cli@17.6.5 && \
 		echo "\033[0;32mCommitlint installed successfully.\033[0m" || \
 		echo "\033[0;31mNode is not installed. Please install Node.js to use commitlint.\033[0m"
 	go mod tidy
-	@echo "\033[0;32mGo modules installed successfully.\033[0m"
+	@echo "\033[0;32mGo modules installed successfully.\033[0m"
+tidy:
+	@echo "\033[0;32mRunning go mod tidy...\033[0m"
+	go mod tidy -v
+	@echo "\033[0;32mVerifying packages...\033[0m"
+	go mod verify

core/model.go

@@ -26,6 +26,7 @@ type Model[T any] struct {
 	Cloned              bool   // Indicates if this model has been cloned at least once
 	pipeline            mongo.Pipeline
 	executor            func(m Model[T], ctx context.Context) any
+	result              any // Pointer to the result of the last executed query. This is still not in full use. Only used for operations such as Populate for now.
 	whereField          string
 	failWith            *error
 	orConditionActive   bool
@@ -40,6 +41,7 @@ type Model[T any] struct {
 	softDeleteEnabled   bool
 	deletedAtFieldName  string
 	triggerExit         chan bool
+	docReflectType      reflect.Type // The reflect type of a sample document of this model
 }
 
 var pluralizeClient = pluralize.NewClient()
@@ -64,6 +66,7 @@ func NewModel[T any](name string, schema Schema) Model[T] {
 		middleware:  &middleware,
 		triggerExit: make(chan bool, 1),
 	}
+	model.preprocess()
 	Models[name] = model
 	onConnectionComplete := func() {
 		model.CreateCollection()
@@ -84,12 +87,11 @@ func NewModel[T any](name string, schema Schema) Model[T] {
 // This method validates the document against the model schema and panics if any errors are found.
 func (m Model[T]) Create(doc T) Model[T] {
 	m.executor = func(m Model[T], ctx context.Context) any {
-		documentToInsert, detailedDocument := enforceSchema(m.Schema, &doc, nil)
-		detailedDocumentEntity := utils.CastBSON[T](detailedDocument)
-		m.middleware.pre.save.run(detailedDocumentEntity)
+		documentToInsert := enforceSchema(m.Schema, &doc, nil)
+		m.middleware.pre.save.run(&documentToInsert)
 		lo.Must(m.Collection().InsertOne(ctx, documentToInsert))
-		m.middleware.post.save.run(detailedDocumentEntity)
-		return detailedDocumentEntity
+		m.middleware.post.save.run(&documentToInsert)
+		return utils.CastBSON[T](documentToInsert)
 	}
 	return m
 }
@@ -104,14 +106,12 @@ func (m Model[T]) CreateMany(docs []T) Model[T] {
 // This method validates the document against the model schema and panics if any errors are found.
 func (m Model[T]) InsertMany(docs []T) Model[T] {
 	m.executor = func(m Model[T], ctx context.Context) any {
-		var documentsToInsert, detailedDocuments []any
+		var documentsToInsert []any
 		for _, doc := range docs {
-			documentToInsert, detailedDocument := enforceSchema(m.Schema, &doc, nil)
-			documentsToInsert = append(documentsToInsert, documentToInsert)
-			detailedDocuments = append(detailedDocuments, detailedDocument)
+			documentsToInsert = append(documentsToInsert, enforceSchema(m.Schema, &doc, nil))
 		}
 		lo.Must(m.Collection().InsertMany(ctx, documentsToInsert))
-		return utils.CastBSONSlice[T](detailedDocuments)
+		return utils.CastBSONSlice[T](documentsToInsert)
 	}
 	return m
 }
@@ -124,8 +124,8 @@ func (m Model[T]) Find(query ...primitive.M) Model[T] {
 		var results []T
 		cursor := lo.Must(m.Collection().Aggregate(ctx, m.pipeline))
 		m.checkConditionsAndPanicForErr(cursor.All(ctx, &results))
-		m.middleware.post.find.run(results)
 		m.checkConditionsAndPanic(results)
+		m.middleware.post.find.run(&results)
 		return results
 	}
 	q := utils.MergedQueryOrDefault(query)

core/model_actions.go

@@ -2,8 +2,6 @@ package elemental
 
 import (
 	"context"
-	"reflect"
-
 	"github.com/elcengine/elemental/utils"
 	"github.com/samber/lo"
 
@@ -31,8 +29,7 @@ func (m Model[T]) Ping(ctx ...context.Context) error {
 
 // Creates or updates the indexes for this model. This method will only create the indexes if they do not exist.
 func (m Model[T]) SyncIndexes(ctx ...context.Context) {
-	var sample [0]T
-	m.Schema.syncIndexes(reflect.TypeOf(sample).Elem(), lo.FromPtr(m.temporaryDatabase), lo.FromPtr(m.temporaryConnection), lo.FromPtr(m.temporaryCollection), ctx...)
+	m.Schema.syncIndexes(m.docReflectType, lo.FromPtr(m.temporaryDatabase), lo.FromPtr(m.temporaryConnection), lo.FromPtr(m.temporaryCollection), ctx...)
 }
 
 // Drops all indexes for this model except the default `_id` index.

core/model_audit.go

@@ -2,11 +2,14 @@ package elemental
 
 import (
 	"context"
-	"github.com/elcengine/elemental/utils"
-	"github.com/samber/lo"
 	"reflect"
 	"time"
 
+	"github.com/elcengine/elemental/utils"
+	"github.com/samber/lo"
+	"github.com/spf13/cast"
+
+	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
@@ -20,6 +23,8 @@ const (
 	AuditTypeDelete AuditType = "delete"
 )
 
+const AuditUserFallback = "System"
+
 type Audit struct {
 	Entity    string      `json:"entity" bson:"entity"`         // The name of the model that was audited.
 	Type      AuditType   `json:"type" bson:"type"`             // The type of operation that was performed (insert, update, delete).
@@ -62,23 +67,21 @@ func (m Model[T]) EnableAuditing(ctx ...context.Context) {
 		q.Exec(context)
 	}
 
-	userFallback := "System"
-
 	m.OnInsert(func(doc T) {
 		execWithModelOpts(AuditModel.Create(Audit{
 			Entity:    m.Name,
 			Type:      AuditTypeInsert,
-			Document:  *utils.ToBSONDoc(doc),
-			User:      lo.CoalesceOrEmpty(utils.Cast[string](context.Value(CtxUser)), userFallback),
+			Document:  utils.CastBSON[bson.M](doc),
+			User:      lo.CoalesceOrEmpty(cast.ToString(context.Value(CtxUser)), AuditUserFallback),
 			CreatedAt: time.Now(),
 		}))
 	}, TriggerOptions{Context: &context, Filter: &primitive.M{"ns.coll": primitive.M{"$eq": m.Collection().Name()}}})
 	m.OnUpdate(func(doc T) {
 		execWithModelOpts(AuditModel.Create(Audit{
 			Entity:    m.Name,
 			Type:      AuditTypeUpdate,
-			Document:  *utils.ToBSONDoc(doc),
-			User:      lo.CoalesceOrEmpty(utils.Cast[string](context.Value(CtxUser)), userFallback),
+			Document:  utils.CastBSON[bson.M](doc),
+			User:      lo.CoalesceOrEmpty(cast.ToString(context.Value(CtxUser)), AuditUserFallback),
 			CreatedAt: time.Now(),
 		}))
 	}, TriggerOptions{Context: &context, Filter: &primitive.M{"ns.coll": primitive.M{"$eq": m.Collection().Name()}}})
@@ -87,7 +90,7 @@ func (m Model[T]) EnableAuditing(ctx ...context.Context) {
 			Entity:    m.Name,
 			Type:      AuditTypeDelete,
 			Document:  map[string]any{"_id": id},
-			User:      lo.CoalesceOrEmpty(utils.Cast[string](context.Value(CtxUser)), userFallback),
+			User:      lo.CoalesceOrEmpty(cast.ToString(context.Value(CtxUser)), AuditUserFallback),
 			CreatedAt: time.Now(),
 		}))
 	}, TriggerOptions{Context: &context, Filter: &primitive.M{"ns.coll": primitive.M{"$eq": m.Collection().Name()}}})

core/model_middleware.go

@@ -3,6 +3,7 @@ package elemental
 import (
 	"github.com/elcengine/elemental/utils"
 
+	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 )
@@ -14,12 +15,13 @@ type listener[T any] struct {
 }
 
 type pre[T any] struct {
-	save             listener[T]
-	updateOne        listener[T]
-	deleteOne        listener[T]
-	deleteMany       listener[T]
-	findOneAndUpdate listener[T]
-	findOneAndDelete listener[T]
+	save              listener[T]
+	updateOne         listener[T]
+	deleteOne         listener[T]
+	deleteMany        listener[T]
+	findOneAndUpdate  listener[T]
+	findOneAndDelete  listener[T]
+	findOneAndReplace listener[T]
 }
 
 type post[T any] struct {
@@ -51,15 +53,15 @@ func (l listener[T]) run(args ...any) {
 	}
 }
 
-func (m Model[T]) PreSave(f func(doc T) bool) {
+func (m Model[T]) PreSave(f func(doc *bson.M) bool) {
 	m.middleware.pre.save.functions = append(m.middleware.pre.save.functions, func(args ...any) bool {
-		return f(args[0].(T))
+		return f(args[0].(*bson.M))
 	})
 }
 
-func (m Model[T]) PostSave(f func(doc T) bool) {
+func (m Model[T]) PostSave(f func(doc *bson.M) bool) {
 	m.middleware.post.save.functions = append(m.middleware.post.save.functions, func(args ...any) bool {
-		return f(args[0].(T))
+		return f(args[0].(*bson.M))
 	})
 }
 
@@ -75,9 +77,9 @@ func (m Model[T]) PostUpdateOne(f func(result *mongo.UpdateResult, err error) bo
 	})
 }
 
-func (m Model[T]) PreDeleteOne(f func(filters primitive.M) bool) {
+func (m Model[T]) PreDeleteOne(f func(filters *primitive.M) bool) {
 	m.middleware.pre.deleteOne.functions = append(m.middleware.pre.deleteOne.functions, func(args ...any) bool {
-		return f(args[0].(primitive.M))
+		return f(args[0].(*primitive.M))
 	})
 }
 
@@ -87,9 +89,9 @@ func (m Model[T]) PostDeleteOne(f func(result *mongo.DeleteResult, err error) bo
 	})
 }
 
-func (m Model[T]) PreDeleteMany(f func(filters primitive.M) bool) {
+func (m Model[T]) PreDeleteMany(f func(filters *primitive.M) bool) {
 	m.middleware.pre.deleteMany.functions = append(m.middleware.pre.deleteMany.functions, func(args ...any) bool {
-		return f(args[0].(primitive.M))
+		return f(args[0].(*primitive.M))
 	})
 }
 
@@ -99,27 +101,27 @@ func (m Model[T]) PostDeleteMany(f func(result *mongo.DeleteResult, err error) b
 	})
 }
 
-func (m Model[T]) PostFind(f func(doc []T) bool) {
+func (m Model[T]) PostFind(f func(doc *[]T) bool) {
 	m.middleware.post.find.functions = append(m.middleware.post.find.functions, func(args ...any) bool {
-		return f(args[0].([]T))
+		return f(args[0].(*[]T))
 	})
 }
 
-func (m Model[T]) PostFindOneAndUpdate(f func(doc *T) bool) {
-	m.middleware.post.findOneAndUpdate.functions = append(m.middleware.post.findOneAndUpdate.functions, func(args ...any) bool {
-		return f(args[0].(*T))
+func (m Model[T]) PreFindOneAndUpdate(f func(filters *primitive.M, doc any) bool) {
+	m.middleware.pre.findOneAndUpdate.functions = append(m.middleware.pre.findOneAndUpdate.functions, func(args ...any) bool {
+		return f(args[0].(*primitive.M), args[1])
 	})
 }
 
-func (m Model[T]) PreFindOneAndUpdate(f func(filters primitive.M) bool) {
-	m.middleware.pre.findOneAndUpdate.functions = append(m.middleware.pre.findOneAndUpdate.functions, func(args ...any) bool {
-		return f(args[0].(primitive.M))
+func (m Model[T]) PostFindOneAndUpdate(f func(doc *T) bool) {
+	m.middleware.post.findOneAndUpdate.functions = append(m.middleware.post.findOneAndUpdate.functions, func(args ...any) bool {
+		return f(args[0].(*T))
 	})
 }
 
-func (m Model[T]) PreFindOneAndDelete(f func(filters primitive.M) bool) {
+func (m Model[T]) PreFindOneAndDelete(f func(filters *primitive.M) bool) {
 	m.middleware.pre.findOneAndDelete.functions = append(m.middleware.pre.findOneAndDelete.functions, func(args ...any) bool {
-		return f(args[0].(primitive.M))
+		return f(args[0].(*primitive.M))
 	})
 }
 
@@ -129,6 +131,12 @@ func (m Model[T]) PostFindOneAndDelete(f func(doc *T) bool) {
 	})
 }
 
+func (m Model[T]) PreFindOneAndReplace(f func(filters *primitive.M, doc any) bool) {
+	m.middleware.pre.findOneAndReplace.functions = append(m.middleware.pre.findOneAndReplace.functions, func(args ...any) bool {
+		return f(args[0].(*primitive.M), args[1])
+	})
+}
+
 func (m Model[T]) PostFindOneAndReplace(f func(doc *T) bool) {
 	m.middleware.post.findOneAndReplace.functions = append(m.middleware.post.findOneAndReplace.functions, func(args ...any) bool {
 		return f(args[0].(*T))