From 21181d8b5cc825255eb2e50404022f6ac9e40056 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 20 Jul 2026 16:38:26 +0300 Subject: [PATCH 1/6] dbutil/reflectscan: optimise scanning and add struct tag option --- dbutil/reflectscan.go | 74 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/dbutil/reflectscan.go b/dbutil/reflectscan.go index 77dc3ac..6cde1d3 100644 --- a/dbutil/reflectscan.go +++ b/dbutil/reflectscan.go @@ -7,24 +7,78 @@ package dbutil import ( + "fmt" "reflect" ) -func reflectScan[T any](row Scannable) (*T, error) { - t := new(T) - val := reflect.ValueOf(t).Elem() - fields := reflect.VisibleFields(val.Type()) - scanInto := make([]any, len(fields)) - for i, field := range fields { - scanInto[i] = val.FieldByIndex(field.Index).Addr().Interface() +func reflectScan[T any]() ConvertRowFn[*T] { + fields := reflect.VisibleFields(reflect.TypeFor[T]()) + return func(row Scannable) (*T, error) { + t := new(T) + val := reflect.ValueOf(t).Elem() + scanInto := make([]any, len(fields)) + for i, field := range fields { + scanInto[i] = val.FieldByIndex(field.Index).Addr().Interface() + } + err := row.Scan(scanInto...) + return t, err } - err := row.Scan(scanInto...) - return t, err +} + +func getFieldMap[T any]() map[string][]int { + fields := reflect.VisibleFields(reflect.TypeFor[T]()) + m := make(map[string][]int, len(fields)) + for _, field := range fields { + sqlName := field.Tag.Get("sql") + if sqlName == "" { + sqlName = field.Name + } + m[sqlName] = field.Index + } + return m +} + +func reflectScanComplicated[T any](rows Rows, err error) (ConvertRowFn[*T], error) { + if err != nil { + return nil, err + } + var fields [][]int + columns, err := rows.Columns() + if err != nil { + return nil, fmt.Errorf("reflectscan: failed to get columns: %w", err) + } + fields = make([][]int, len(columns)) + fieldMap := getFieldMap[T]() + var ok bool + for i, col := range columns { + fields[i], ok = fieldMap[col] + if !ok { + return nil, fmt.Errorf("reflectscan: column %q does not match any struct field", col) + } + } + return func(row Scannable) (*T, error) { + t := new(T) + val := reflect.ValueOf(t).Elem() + scanInto := make([]any, len(fields)) + for i, idx := range fields { + scanInto[i] = val.FieldByIndex(idx).Addr().Interface() + } + err := row.Scan(scanInto...) + return t, err + }, nil } // NewSimpleReflectRowIter creates a new RowIter that uses reflection to scan rows into the given type. // // This is a simplified implementation that always scans to all struct fields. It does not support any kind of struct tags. func NewSimpleReflectRowIter[T any](rows Rows, err error) RowIter[*T] { - return ConvertRowFn[*T](reflectScan[T]).NewRowIter(rows, err) + return reflectScan[T]().NewRowIter(rows, err) +} + +// NewComplicatedReflectRowIter creates a new RowIter that uses reflection to scan rows into the given type. +// +// This will use the `sql` struct tag. The column names returned by the db must match an explicit struct tag exactly. +func NewComplicatedReflectRowIter[T any](rows Rows, err error) RowIter[*T] { + fn, err := reflectScanComplicated[T](rows, err) + return fn.NewRowIter(rows, err) } From 2107afbb1495ebb09b4173cb95c09059de1a7724 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 20 Jul 2026 16:44:37 +0300 Subject: [PATCH 2/6] Make struct tag customizable --- dbutil/reflectscan.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/dbutil/reflectscan.go b/dbutil/reflectscan.go index 6cde1d3..1f5b0f2 100644 --- a/dbutil/reflectscan.go +++ b/dbutil/reflectscan.go @@ -11,7 +11,7 @@ import ( "reflect" ) -func reflectScan[T any]() ConvertRowFn[*T] { +func MakeSimpleReflectScanner[T any]() ConvertRowFn[*T] { fields := reflect.VisibleFields(reflect.TypeFor[T]()) return func(row Scannable) (*T, error) { t := new(T) @@ -25,11 +25,11 @@ func reflectScan[T any]() ConvertRowFn[*T] { } } -func getFieldMap[T any]() map[string][]int { +func getFieldMap[T any](structTag string) map[string][]int { fields := reflect.VisibleFields(reflect.TypeFor[T]()) m := make(map[string][]int, len(fields)) for _, field := range fields { - sqlName := field.Tag.Get("sql") + sqlName := field.Tag.Get(structTag) if sqlName == "" { sqlName = field.Name } @@ -38,7 +38,9 @@ func getFieldMap[T any]() map[string][]int { return m } -func reflectScanComplicated[T any](rows Rows, err error) (ConvertRowFn[*T], error) { +const defaultReflectStructTag = "column" + +func makeReflectScanner[T any](rows Rows, err error, structTag string) (ConvertRowFn[*T], error) { if err != nil { return nil, err } @@ -48,7 +50,7 @@ func reflectScanComplicated[T any](rows Rows, err error) (ConvertRowFn[*T], erro return nil, fmt.Errorf("reflectscan: failed to get columns: %w", err) } fields = make([][]int, len(columns)) - fieldMap := getFieldMap[T]() + fieldMap := getFieldMap[T](structTag) var ok bool for i, col := range columns { fields[i], ok = fieldMap[col] @@ -72,13 +74,17 @@ func reflectScanComplicated[T any](rows Rows, err error) (ConvertRowFn[*T], erro // // This is a simplified implementation that always scans to all struct fields. It does not support any kind of struct tags. func NewSimpleReflectRowIter[T any](rows Rows, err error) RowIter[*T] { - return reflectScan[T]().NewRowIter(rows, err) + return MakeSimpleReflectScanner[T]().NewRowIter(rows, err) } -// NewComplicatedReflectRowIter creates a new RowIter that uses reflection to scan rows into the given type. +// NewReflectRowIter creates a new RowIter that uses reflection to scan rows into the given type. // -// This will use the `sql` struct tag. The column names returned by the db must match an explicit struct tag exactly. -func NewComplicatedReflectRowIter[T any](rows Rows, err error) RowIter[*T] { - fn, err := reflectScanComplicated[T](rows, err) +// This will use the `column` struct tag. The column names returned by the db must match an explicit struct tag exactly. +func NewReflectRowIter[T any](rows Rows, err error) RowIter[*T] { + return NewReflectRowIterWithTag[T](rows, err, defaultReflectStructTag) +} + +func NewReflectRowIterWithTag[T any](rows Rows, err error, structTag string) RowIter[*T] { + fn, err := makeReflectScanner[T](rows, err, structTag) return fn.NewRowIter(rows, err) } From 88fe055d614e0833644da4affb5e35891118b7eb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Jul 2026 19:14:32 +0300 Subject: [PATCH 3/6] Add more options --- dbutil/reflectscan.go | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/dbutil/reflectscan.go b/dbutil/reflectscan.go index 1f5b0f2..899c765 100644 --- a/dbutil/reflectscan.go +++ b/dbutil/reflectscan.go @@ -40,21 +40,33 @@ func getFieldMap[T any](structTag string) map[string][]int { const defaultReflectStructTag = "column" -func makeReflectScanner[T any](rows Rows, err error, structTag string) (ConvertRowFn[*T], error) { +type ReflectScanOptions struct { + StructTag string + IgnoreUnknown bool +} + +type noopScan struct{} + +func (*noopScan) Scan(_ any) error { + return nil +} + +var noopScanVal = &noopScan{} + +func makeReflectScanner[T any](rows Rows, err error, opts ReflectScanOptions) (ConvertRowFn[*T], error) { if err != nil { return nil, err } - var fields [][]int columns, err := rows.Columns() if err != nil { return nil, fmt.Errorf("reflectscan: failed to get columns: %w", err) } - fields = make([][]int, len(columns)) - fieldMap := getFieldMap[T](structTag) + fieldMap := getFieldMap[T](opts.StructTag) + fields := make([][]int, len(columns)) var ok bool for i, col := range columns { fields[i], ok = fieldMap[col] - if !ok { + if !ok && !opts.IgnoreUnknown { return nil, fmt.Errorf("reflectscan: column %q does not match any struct field", col) } } @@ -63,7 +75,11 @@ func makeReflectScanner[T any](rows Rows, err error, structTag string) (ConvertR val := reflect.ValueOf(t).Elem() scanInto := make([]any, len(fields)) for i, idx := range fields { - scanInto[i] = val.FieldByIndex(idx).Addr().Interface() + if idx == nil { + scanInto[i] = noopScanVal + } else { + scanInto[i] = val.FieldByIndex(idx).Addr().Interface() + } } err := row.Scan(scanInto...) return t, err @@ -81,10 +97,10 @@ func NewSimpleReflectRowIter[T any](rows Rows, err error) RowIter[*T] { // // This will use the `column` struct tag. The column names returned by the db must match an explicit struct tag exactly. func NewReflectRowIter[T any](rows Rows, err error) RowIter[*T] { - return NewReflectRowIterWithTag[T](rows, err, defaultReflectStructTag) + return NewReflectRowIterWithOptions[T](rows, err, ReflectScanOptions{StructTag: defaultReflectStructTag}) } -func NewReflectRowIterWithTag[T any](rows Rows, err error, structTag string) RowIter[*T] { - fn, err := makeReflectScanner[T](rows, err, structTag) +func NewReflectRowIterWithOptions[T any](rows Rows, err error, opts ReflectScanOptions) RowIter[*T] { + fn, err := makeReflectScanner[T](rows, err, opts) return fn.NewRowIter(rows, err) } From bd41e7995c556c1ba07108e3360f33cda45305ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Jul 2026 19:19:02 +0300 Subject: [PATCH 4/6] Split initialization function --- dbutil/reflectscan.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dbutil/reflectscan.go b/dbutil/reflectscan.go index 899c765..31c9dba 100644 --- a/dbutil/reflectscan.go +++ b/dbutil/reflectscan.go @@ -53,10 +53,7 @@ func (*noopScan) Scan(_ any) error { var noopScanVal = &noopScan{} -func makeReflectScanner[T any](rows Rows, err error, opts ReflectScanOptions) (ConvertRowFn[*T], error) { - if err != nil { - return nil, err - } +func initReflectScan[T any](rows Rows, opts ReflectScanOptions) ([][]int, error) { columns, err := rows.Columns() if err != nil { return nil, fmt.Errorf("reflectscan: failed to get columns: %w", err) @@ -70,6 +67,17 @@ func makeReflectScanner[T any](rows Rows, err error, opts ReflectScanOptions) (C return nil, fmt.Errorf("reflectscan: column %q does not match any struct field", col) } } + return fields, nil +} + +func makeReflectScanner[T any](rows Rows, err error, opts ReflectScanOptions) (ConvertRowFn[*T], error) { + if err != nil { + return nil, err + } + fields, err := initReflectScan[T](rows, opts) + if err != nil { + return nil, err + } return func(row Scannable) (*T, error) { t := new(T) val := reflect.ValueOf(t).Elem() From 3b37dd273dd9133ba0dcbd77a1a9090221beb8af Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 24 Jul 2026 11:16:35 +0300 Subject: [PATCH 5/6] Expose manual reflect scan construction --- dbutil/reflectscan.go | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/dbutil/reflectscan.go b/dbutil/reflectscan.go index 31c9dba..e43ef47 100644 --- a/dbutil/reflectscan.go +++ b/dbutil/reflectscan.go @@ -53,11 +53,15 @@ func (*noopScan) Scan(_ any) error { var noopScanVal = &noopScan{} -func initReflectScan[T any](rows Rows, opts ReflectScanOptions) ([][]int, error) { +func initReflectScanWithRows[T any](rows Rows, opts ReflectScanOptions) ([][]int, error) { columns, err := rows.Columns() if err != nil { return nil, fmt.Errorf("reflectscan: failed to get columns: %w", err) } + return initReflectScanWithColumns[T](columns, opts) +} + +func initReflectScanWithColumns[T any](columns []string, opts ReflectScanOptions) ([][]int, error) { fieldMap := getFieldMap[T](opts.StructTag) fields := make([][]int, len(columns)) var ok bool @@ -70,14 +74,7 @@ func initReflectScan[T any](rows Rows, opts ReflectScanOptions) ([][]int, error) return fields, nil } -func makeReflectScanner[T any](rows Rows, err error, opts ReflectScanOptions) (ConvertRowFn[*T], error) { - if err != nil { - return nil, err - } - fields, err := initReflectScan[T](rows, opts) - if err != nil { - return nil, err - } +func makeReflectScannerWithFields[T any](fields [][]int) ConvertRowFn[*T] { return func(row Scannable) (*T, error) { t := new(T) val := reflect.ValueOf(t).Elem() @@ -91,7 +88,32 @@ func makeReflectScanner[T any](rows Rows, err error, opts ReflectScanOptions) (C } err := row.Scan(scanInto...) return t, err - }, nil + } +} + +func MakeReflectScanner[T any](columns []string, opts ...ReflectScanOptions) (ConvertRowFn[*T], error) { + var opt ReflectScanOptions + if len(opts) == 1 { + opt = opts[0] + } else if len(opts) > 1 { + return nil, fmt.Errorf("reflectscan: only one ReflectScanOptions is allowed") + } + fields, err := initReflectScanWithColumns[T](columns, opt) + if err != nil { + return nil, err + } + return makeReflectScannerWithFields[T](fields), nil +} + +func makeReflectScanner[T any](rows Rows, err error, opts ReflectScanOptions) (ConvertRowFn[*T], error) { + if err != nil { + return nil, err + } + fields, err := initReflectScanWithRows[T](rows, opts) + if err != nil { + return nil, err + } + return makeReflectScannerWithFields[T](fields), nil } // NewSimpleReflectRowIter creates a new RowIter that uses reflection to scan rows into the given type. From 9ac7165bdef662868f3493ac139b5770588d0909 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 24 Jul 2026 11:20:12 +0300 Subject: [PATCH 6/6] Add docstrings --- dbutil/reflectscan.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dbutil/reflectscan.go b/dbutil/reflectscan.go index e43ef47..f2521d2 100644 --- a/dbutil/reflectscan.go +++ b/dbutil/reflectscan.go @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Tulir Asokan +// Copyright (c) 2026 Tulir Asokan // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -11,6 +11,9 @@ import ( "reflect" ) +// MakeSimpleReflectScanner creates a ConvertRowFn that uses reflection to scan rows into the given type. +// +// This is a simplified implementation that always scans to all struct fields. It does not support any kind of struct tags. func MakeSimpleReflectScanner[T any]() ConvertRowFn[*T] { fields := reflect.VisibleFields(reflect.TypeFor[T]()) return func(row Scannable) (*T, error) { @@ -91,6 +94,9 @@ func makeReflectScannerWithFields[T any](fields [][]int) ConvertRowFn[*T] { } } +// MakeReflectScanner creates a ConvertRowFn that uses reflection to scan rows into the given type. +// +// To detect column names automatically, use [NewReflectRowIter] instead. func MakeReflectScanner[T any](columns []string, opts ...ReflectScanOptions) (ConvertRowFn[*T], error) { var opt ReflectScanOptions if len(opts) == 1 { @@ -126,6 +132,7 @@ func NewSimpleReflectRowIter[T any](rows Rows, err error) RowIter[*T] { // NewReflectRowIter creates a new RowIter that uses reflection to scan rows into the given type. // // This will use the `column` struct tag. The column names returned by the db must match an explicit struct tag exactly. +// Use [NewReflectRowIterWithOptions] to customize the struct tag or ignore unknown columns. func NewReflectRowIter[T any](rows Rows, err error) RowIter[*T] { return NewReflectRowIterWithOptions[T](rows, err, ReflectScanOptions{StructTag: defaultReflectStructTag}) }