From 452e1d22801c69f7a51ea063b3c5dee93ace5a14 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Thu, 2 Oct 2025 13:45:02 +0200 Subject: [PATCH 1/3] Support alternative sortBy field --- v2/dbutils/ops/gen/README.md | 10 +- v2/dbutils/ops/gen/generator.go | 28 +++- v2/dbutils/ops/gen/generator_test.go | 164 ++++++++++++++++++++++ v2/dbutils/ops/gen/tst/tst.go | 4 +- v2/dbutils/ops/gen/tst/tst_filters.gen.go | 7 +- 5 files changed, 202 insertions(+), 11 deletions(-) diff --git a/v2/dbutils/ops/gen/README.md b/v2/dbutils/ops/gen/README.md index f45fb9d..622e775 100644 --- a/v2/dbutils/ops/gen/README.md +++ b/v2/dbutils/ops/gen/README.md @@ -23,20 +23,24 @@ Add `db:filter` comments to fields that should be filterable: type ListDCRsRequest struct { // db:filter bob_gen.ColumnNames.DCRS.Type Type string `query:"type"` - // db:filter bob_gen.ColumnNames.DCRS.Status - Status string `query:"status"` + // db:filter bob_gen.ColumnNames.DCRS.Status sortBy "dcrs.status_order" + Status string `query:"status"` // <----- sortBy is optional, to specify a different column for sorting // db:filter bob_gen.ColumnNames.DCRS.CreatedBy CreatedBy *string `query:"created_by"` // db:filter bob_gen.ColumnNames.DCRS.Tags Tags []string `query:"tags"` - + // Regular fields without filter comments are ignored Limit int `query:"limit"` Offset int `query:"offset"` + + Sort []string `query:"sort"` // <----- this field is referenced by sortField } ``` Of course, this is assuming Huma. There is no support for Goa, sorry. +**Note on `sortBy`**: When you specify `sortBy` on a field, the generator creates a separate `SortColumnsMap` that maps query parameters to their sort columns. This is useful when the column you want to sort by is different from the column you filter on. Fields without `sortBy` will use their filter column for sorting. + ### 2. Add go generate directive diff --git a/v2/dbutils/ops/gen/generator.go b/v2/dbutils/ops/gen/generator.go index 8fb0080..b70d119 100644 --- a/v2/dbutils/ops/gen/generator.go +++ b/v2/dbutils/ops/gen/generator.go @@ -19,6 +19,7 @@ type FilterField struct { QueryParam string // Query parameter name from struct tag Type string // Field type Having bool // Whether this filter should use HAVING instead of WHERE + SortBy string // Optional: Database column name for sorting (from sortBy comment) } // StructInfo contains information about a struct that needs filter generation @@ -209,7 +210,8 @@ func (g *Generator) parseImportSpec(spec string) string { return spec } -var filterCommentRegex = regexp.MustCompile(`//\s*db:filter\s+(.+?)(?:\s+having)?$`) +var filterCommentRegex = regexp.MustCompile(`//\s*db:filter\s+(.+?)(?:\s+having)?(?:\s+sortBy\s+(.+?))?$`) +var sortByCommentRegex = regexp.MustCompile(`//\s*db:filter\s+.+?\s+sortBy\s+(.+?)(?:\s+having)?$`) // parseStruct extracts filter field information from a struct func (g *Generator) parseStruct(name string, structType *ast.StructType) StructInfo { @@ -227,13 +229,23 @@ func (g *Generator) parseStruct(name string, structType *ast.StructType) StructI // Check for db:filter comment var column string var having bool + var sortBy string for _, comment := range field.Doc.List { // Check if comment contains "having" parameter having = strings.Contains(comment.Text, " having") - + matches := filterCommentRegex.FindStringSubmatch(comment.Text) if len(matches) > 1 { column = strings.TrimSpace(matches[1]) + } + + // Check for sortBy parameter + sortByMatches := sortByCommentRegex.FindStringSubmatch(comment.Text) + if len(sortByMatches) > 1 { + sortBy = strings.TrimSpace(sortByMatches[1]) + } + + if column != "" { break } } @@ -263,6 +275,7 @@ func (g *Generator) parseStruct(name string, structType *ast.StructType) StructI Type: fieldType, QueryParam: queryParam, Having: having, + SortBy: sortBy, }) } } @@ -374,12 +387,17 @@ import ( {{end}} {{range .AdditionalImports}} {{.}} {{end}}){{$lib := .FilterType}} -{{range .Structs}}{{$receiver := .ReceiverName}}{{$structName := .Name}} +{{range .Structs}}{{$receiver := .ReceiverName}}{{$structName := .Name}}{{$hasSortBy := false}}{{range .Fields}}{{if ne .SortBy ""}}{{$hasSortBy = true}}{{end}}{{end}} // {{.Name}}ColumnsMap is a FilterMap mapping filter names to DB columns // DO NOT EDIT: This var is generated by go-libs/v2/dbutils/ops/gen/cmd var {{.Name}}ColumnsMap = {{if eq $lib "bob"}}bobops.NewBobFilterMap{{else if eq $lib "boiler"}}boilerops.NewBoilFilterMap{{end}}(map[string]string{ {{range .Fields}}"{{.QueryParam}}": {{.Column}},{{end}} -}) +}){{if $hasSortBy}} +// {{.Name}}SortColumnsMap is a FilterMap mapping sort parameter names to DB columns for sorting +// DO NOT EDIT: This var is generated by go-libs/v2/dbutils/ops/gen/cmd +var {{.Name}}SortColumnsMap = {{if eq $lib "bob"}}bobops.NewBobFilterMap{{else if eq $lib "boiler"}}boilerops.NewBoilFilterMap{{end}}(map[string]string{ + {{range .Fields}}{{if ne .SortBy ""}}"{{.QueryParam}}": {{.SortBy}},{{else}}"{{.QueryParam}}": {{.Column}},{{end}}{{end}} +}){{end}} // AddFilters adds database filters based on the struct fields with db:filter comments // DO NOT EDIT: This func is generated by go-libs/v2/dbutils/ops/gen/cmd func ({{.ReceiverName}} *{{.Name}}) AddFilters(q {{if eq $lib "bob"}}*[]bob.Mod[*dialect.SelectQuery]{{else if eq $lib "boiler"}}*[]qm.QueryMod{{end}}) error { @@ -436,7 +454,7 @@ func ({{.ReceiverName}} *{{.Name}}) AddFilters(q {{if eq $lib "bob"}}*[]bob.Mod[ {{if ne .SortField ""}} // AddSorting adds the result of ParseSorting to a given query func ({{.ReceiverName}} *{{.Name}}) AddSorting(query {{if eq $lib "bob"}}*[]bob.Mod[*dialect.SelectQuery]{{else if eq $lib "boiler"}}*[]qm.QueryMod{{end}}) error { - return {{$structName}}ColumnsMap.AddSorting(query, {{$receiver}}.{{.SortField}}) + {{if $hasSortBy}}return {{$structName}}SortColumnsMap.AddSorting(query, {{$receiver}}.{{.SortField}}){{else}}return {{$structName}}ColumnsMap.AddSorting(query, {{$receiver}}.{{.SortField}}){{end}} } {{end}} {{end}} diff --git a/v2/dbutils/ops/gen/generator_test.go b/v2/dbutils/ops/gen/generator_test.go index 1eeefb7..5817606 100644 --- a/v2/dbutils/ops/gen/generator_test.go +++ b/v2/dbutils/ops/gen/generator_test.go @@ -765,3 +765,167 @@ type ComplexRequest struct { assert.Contains(t, generatedCode, `ParseFilter(cond, "DATE_TRUNC('day', created_at)", op, rawValue, false)`) assert.Contains(t, generatedCode, `ParseFilter(cond, "EXTRACT(EPOCH FROM NOW() - created_at)", op, rawValue, false)`) } + +func TestGenerator_SortByFieldMapping(t *testing.T) { + tmpDir := t.TempDir() + + // Create test input file with sortBy mappings + testContent := `package requests + +// db:filter +// db:filter sortField Sort +type ProductsRequest struct { + // db:filter bob_gen.ColumnNames.Products.Name sortBy "products.name" + ProductName string ` + "`query:\"product_name\"`" + ` + // db:filter bob_gen.ColumnNames.Products.Price sortBy "products.price" + Price string ` + "`query:\"price\"`" + ` + // db:filter bob_gen.ColumnNames.Products.Category + Category string ` + "`query:\"category\"`" + ` + Sort []string ` + "`query:\"sort\"`" + ` +}` + + inputFile := filepath.Join(tmpDir, "requests.go") + err := os.WriteFile(inputFile, []byte(testContent), 0644) + require.NoError(t, err) + + generator := NewGenerator("requests", tmpDir, "bob") + + // Parse the file to check struct info + structs, err := generator.parseFile(inputFile) + require.NoError(t, err) + require.Len(t, structs, 1) + + // Verify that sortBy is extracted correctly + fields := structs[0].Fields + require.Len(t, fields, 3) + + assert.Equal(t, "ProductName", fields[0].Name) + assert.Equal(t, `"products.name"`, fields[0].SortBy) + + assert.Equal(t, "Price", fields[1].Name) + assert.Equal(t, `"products.price"`, fields[1].SortBy) + + assert.Equal(t, "Category", fields[2].Name) + assert.Equal(t, "", fields[2].SortBy) // No sortBy specified + + // Generate code and verify the SortColumnsMap is created + err = generator.GenerateFromFile(inputFile) + require.NoError(t, err) + + outputFile := filepath.Join(tmpDir, "requests_filters.gen.go") + content, err := os.ReadFile(outputFile) + require.NoError(t, err) + + generatedCode := string(content) + + // Should contain both ColumnsMap and SortColumnsMap + assert.Contains(t, generatedCode, "var ProductsRequestColumnsMap") + assert.Contains(t, generatedCode, "var ProductsRequestSortColumnsMap") + + // ColumnsMap should use the regular columns + assert.Contains(t, generatedCode, `"product_name": bob_gen.ColumnNames.Products.Name`) + assert.Contains(t, generatedCode, `"price": bob_gen.ColumnNames.Products.Price`) + assert.Contains(t, generatedCode, `"category": bob_gen.ColumnNames.Products.Category`) + + // SortColumnsMap should use sortBy columns where specified + assert.Contains(t, generatedCode, `ProductsRequestSortColumnsMap`) + assert.Contains(t, generatedCode, `"product_name": "products.name"`) + assert.Contains(t, generatedCode, `"price": "products.price"`) + assert.Contains(t, generatedCode, `"category": bob_gen.ColumnNames.Products.Category`) // Falls back to filter column + + // AddSorting should use SortColumnsMap + assert.Contains(t, generatedCode, "func (p *ProductsRequest) AddSorting") + assert.Contains(t, generatedCode, "ProductsRequestSortColumnsMap.AddSorting(query, p.Sort)") +} + +func TestGenerator_SortByWithHaving(t *testing.T) { + tmpDir := t.TempDir() + + // Create test input file with sortBy and having + testContent := `package requests + +// db:filter +// db:filter sortField Sort +type AggregatesRequest struct { + // db:filter "COUNT(*)" having sortBy "count_value" + Count string ` + "`query:\"count\"`" + ` + // db:filter bob_gen.ColumnNames.Users.Name + Name string ` + "`query:\"name\"`" + ` + Sort []string ` + "`query:\"sort\"`" + ` +}` + + inputFile := filepath.Join(tmpDir, "requests.go") + err := os.WriteFile(inputFile, []byte(testContent), 0644) + require.NoError(t, err) + + generator := NewGenerator("requests", tmpDir, "bob") + + // Parse and verify + structs, err := generator.parseFile(inputFile) + require.NoError(t, err) + require.Len(t, structs, 1) + + fields := structs[0].Fields + require.Len(t, fields, 2) + + assert.Equal(t, "Count", fields[0].Name) + assert.Equal(t, `"count_value"`, fields[0].SortBy) + assert.True(t, fields[0].Having) + + assert.Equal(t, "Name", fields[1].Name) + assert.Equal(t, "", fields[1].SortBy) + assert.False(t, fields[1].Having) + + // Generate and verify + err = generator.GenerateFromFile(inputFile) + require.NoError(t, err) + + outputFile := filepath.Join(tmpDir, "requests_filters.gen.go") + content, err := os.ReadFile(outputFile) + require.NoError(t, err) + + generatedCode := string(content) + + // Should have SortColumnsMap since sortBy is present + assert.Contains(t, generatedCode, "var AggregatesRequestSortColumnsMap") + assert.Contains(t, generatedCode, `"count": "count_value"`) + assert.Contains(t, generatedCode, `"name": bob_gen.ColumnNames.Users.Name`) +} + +func TestGenerator_NoSortByNoSortColumnsMap(t *testing.T) { + tmpDir := t.TempDir() + + // Create test input file without any sortBy + testContent := `package requests + +// db:filter +// db:filter sortField Sort +type SimpleRequest struct { + // db:filter bob_gen.ColumnNames.Users.Name + Name string ` + "`query:\"name\"`" + ` + // db:filter bob_gen.ColumnNames.Users.Email + Email string ` + "`query:\"email\"`" + ` + Sort []string ` + "`query:\"sort\"`" + ` +}` + + inputFile := filepath.Join(tmpDir, "requests.go") + err := os.WriteFile(inputFile, []byte(testContent), 0644) + require.NoError(t, err) + + generator := NewGenerator("requests", tmpDir, "bob") + + err = generator.GenerateFromFile(inputFile) + require.NoError(t, err) + + outputFile := filepath.Join(tmpDir, "requests_filters.gen.go") + content, err := os.ReadFile(outputFile) + require.NoError(t, err) + + generatedCode := string(content) + + // Should NOT have SortColumnsMap when no sortBy is specified + assert.NotContains(t, generatedCode, "SortColumnsMap") + + // AddSorting should use regular ColumnsMap + assert.Contains(t, generatedCode, "SimpleRequestColumnsMap.AddSorting(query, s.Sort)") +} diff --git a/v2/dbutils/ops/gen/tst/tst.go b/v2/dbutils/ops/gen/tst/tst.go index b8a1c1f..d02c235 100644 --- a/v2/dbutils/ops/gen/tst/tst.go +++ b/v2/dbutils/ops/gen/tst/tst.go @@ -18,9 +18,9 @@ var tablename = struct { // db:filter sortField Sort type TestStruct struct { Sortable - // db:filter "stuff" + // db:filter "stuff" sortBy "sorted_stuff" Test string `query:"test"` - // db:filter fmt.Sprintf("heee") + // db:filter fmt.Sprintf("heee") sortBy fmt.Sprintf("sorted_heee") Test2 *string `query:"test2"` // db:filter "EEEI" Test3 []string `query:"test3"` diff --git a/v2/dbutils/ops/gen/tst/tst_filters.gen.go b/v2/dbutils/ops/gen/tst/tst_filters.gen.go index ab140fe..43d7923 100644 --- a/v2/dbutils/ops/gen/tst/tst_filters.gen.go +++ b/v2/dbutils/ops/gen/tst/tst_filters.gen.go @@ -16,6 +16,11 @@ import ( var TestStructColumnsMap = bobops.NewBobFilterMap(map[string]string{ "test": "stuff","test2": fmt.Sprintf("heee"),"test3": "EEEI","test4": "group_col","test5": "having_ptr_col","test6": "having_array_col","test7": "(CASE WHEN bom.pn = bom.enditem THEN 1 END)","test8": "COALESCE(users.name, users.email, 'Unknown')","test9": "COUNT(*) FILTER (WHERE status = 'active')","test10": "DATE_TRUNC('day', created_at)","test11": simple_column,"test12": tablename.column_name, }) +// TestStructSortColumnsMap is a FilterMap mapping sort parameter names to DB columns for sorting +// DO NOT EDIT: This var is generated by go-libs/v2/dbutils/ops/gen/cmd +var TestStructSortColumnsMap = bobops.NewBobFilterMap(map[string]string{ + "test": "sorted_stuff","test2": fmt.Sprintf("sorted_heee"),"test3": "EEEI","test4": "group_col","test5": "having_ptr_col","test6": "having_array_col","test7": "(CASE WHEN bom.pn = bom.enditem THEN 1 END)","test8": "COALESCE(users.name, users.email, 'Unknown')","test9": "COUNT(*) FILTER (WHERE status = 'active')","test10": "DATE_TRUNC('day', created_at)","test11": simple_column,"test12": tablename.column_name, +}) // AddFilters adds database filters based on the struct fields with db:filter comments // DO NOT EDIT: This func is generated by go-libs/v2/dbutils/ops/gen/cmd func (t *TestStruct) AddFilters(q *[]bob.Mod[*dialect.SelectQuery]) error { @@ -184,7 +189,7 @@ func (t *TestStruct) AddFilters(q *[]bob.Mod[*dialect.SelectQuery]) error { // AddSorting adds the result of ParseSorting to a given query func (t *TestStruct) AddSorting(query *[]bob.Mod[*dialect.SelectQuery]) error { - return TestStructColumnsMap.AddSorting(query, t.Sort) + return TestStructSortColumnsMap.AddSorting(query, t.Sort) } From 015c0c3fdd11d582ace5e8661e61d6a8fe6967fa Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Thu, 2 Oct 2025 13:51:16 +0200 Subject: [PATCH 2/3] Handle query struct tags with commas --- v2/dbutils/ops/gen/README.md | 2 +- v2/dbutils/ops/gen/generator.go | 7 +++- v2/dbutils/ops/gen/generator_test.go | 49 ++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/v2/dbutils/ops/gen/README.md b/v2/dbutils/ops/gen/README.md index 622e775..8757456 100644 --- a/v2/dbutils/ops/gen/README.md +++ b/v2/dbutils/ops/gen/README.md @@ -37,11 +37,11 @@ type ListDCRsRequest struct { Sort []string `query:"sort"` // <----- this field is referenced by sortField } ``` + Of course, this is assuming Huma. There is no support for Goa, sorry. **Note on `sortBy`**: When you specify `sortBy` on a field, the generator creates a separate `SortColumnsMap` that maps query parameters to their sort columns. This is useful when the column you want to sort by is different from the column you filter on. Fields without `sortBy` will use their filter column for sorting. - ### 2. Add go generate directive Add this line to the top of your model files (it will also work in main.go, only a bit slower): diff --git a/v2/dbutils/ops/gen/generator.go b/v2/dbutils/ops/gen/generator.go index b70d119..8fbfc32 100644 --- a/v2/dbutils/ops/gen/generator.go +++ b/v2/dbutils/ops/gen/generator.go @@ -308,7 +308,12 @@ func (g *Generator) extractQueryTag(tag string) string { queryRegex := regexp.MustCompile(`query:"([^"]*)"`) matches := queryRegex.FindStringSubmatch(tag) if len(matches) > 1 { - return matches[1] + // Split by comma and return the first value (standard struct tag format) + value := matches[1] + if parts := strings.Split(value, ","); len(parts) > 0 { + return strings.TrimSpace(parts[0]) + } + return value } return "" diff --git a/v2/dbutils/ops/gen/generator_test.go b/v2/dbutils/ops/gen/generator_test.go index 5817606..248235e 100644 --- a/v2/dbutils/ops/gen/generator_test.go +++ b/v2/dbutils/ops/gen/generator_test.go @@ -929,3 +929,52 @@ type SimpleRequest struct { // AddSorting should use regular ColumnsMap assert.Contains(t, generatedCode, "SimpleRequestColumnsMap.AddSorting(query, s.Sort)") } + +func TestGenerator_QueryTagWithComma(t *testing.T) { + tmpDir := t.TempDir() + + // Create test input file with comma-separated query tags + testContent := `package requests + +// db:filter +type TestRequest struct { + // db:filter bob_gen.ColumnNames.Users.Name + Name string ` + "`query:\"name,omitempty\"`" + ` + // db:filter bob_gen.ColumnNames.Users.Email + Email string ` + "`query:\"email,required\"`" + ` +}` + + inputFile := filepath.Join(tmpDir, "requests.go") + err := os.WriteFile(inputFile, []byte(testContent), 0644) + require.NoError(t, err) + + generator := NewGenerator("requests", tmpDir, "bob") + + // Parse and verify + structs, err := generator.parseFile(inputFile) + require.NoError(t, err) + require.Len(t, structs, 1) + + fields := structs[0].Fields + require.Len(t, fields, 2) + + // Should extract only the first part before the comma + assert.Equal(t, "name", fields[0].QueryParam) + assert.Equal(t, "email", fields[1].QueryParam) + + // Generate and verify + err = generator.GenerateFromFile(inputFile) + require.NoError(t, err) + + outputFile := filepath.Join(tmpDir, "requests_filters.gen.go") + content, err := os.ReadFile(outputFile) + require.NoError(t, err) + + generatedCode := string(content) + + // Should use the name without the comma-separated options + assert.Contains(t, generatedCode, `"name": bob_gen.ColumnNames.Users.Name`) + assert.Contains(t, generatedCode, `"email": bob_gen.ColumnNames.Users.Email`) + assert.NotContains(t, generatedCode, "omitempty") + assert.NotContains(t, generatedCode, "required") +} From 3a92a7af351905535340cd3748a4b39f76a07795 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Thu, 2 Oct 2025 13:57:28 +0200 Subject: [PATCH 3/3] Fix CI --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 956ad94..b9728fc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,3 +18,4 @@ jobs: - name: Test run: go test -v ./... + working-directory: v2