From 5522cfeb4e0c70c6966998c619decc03089ba6fa Mon Sep 17 00:00:00 2001 From: Gerfey Date: Mon, 28 Jul 2025 21:18:47 +0700 Subject: [PATCH 01/17] feat: added Kafka transport --- builder/builder.go | 2 + config/config.go | 1 + .../kafka_transport/handler/hello_handler.go | 20 +++ examples/kafka_transport/kafka_transport.go | 65 +++++++++ examples/kafka_transport/message/hello.go | 9 ++ examples/kafka_transport/messenger.yaml | 19 +++ go.mod | 5 +- go.sum | 55 ++++++++ transport/amqp/adapters.go | 118 ---------------- transport/kafka/config.go | 62 ++++++++ transport/kafka/consumer.go | 132 ++++++++++++++++++ transport/kafka/factory.go | 40 ++++++ transport/kafka/producer.go | 55 ++++++++ transport/kafka/transport.go | 53 +++++++ transport/sync/transport.go | 4 +- 15 files changed, 519 insertions(+), 121 deletions(-) create mode 100644 examples/kafka_transport/handler/hello_handler.go create mode 100644 examples/kafka_transport/kafka_transport.go create mode 100644 examples/kafka_transport/message/hello.go create mode 100644 examples/kafka_transport/messenger.yaml delete mode 100644 transport/amqp/adapters.go create mode 100644 transport/kafka/config.go create mode 100644 transport/kafka/consumer.go create mode 100644 transport/kafka/factory.go create mode 100644 transport/kafka/producer.go create mode 100644 transport/kafka/transport.go diff --git a/builder/builder.go b/builder/builder.go index 8930ca1..cea9488 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -21,6 +21,7 @@ import ( "github.com/gerfey/messenger/transport" "github.com/gerfey/messenger/transport/amqp" "github.com/gerfey/messenger/transport/inmemory" + "github.com/gerfey/messenger/transport/kafka" "github.com/gerfey/messenger/transport/sync" ) @@ -45,6 +46,7 @@ func NewBuilder(cfg *config.MessengerConfig, logger *slog.Logger) api.Builder { amqp.NewTransportFactory(logger, resolver), inmemory.NewTransportFactory(logger, resolver), sync.NewTransportFactory(logger, busLocator), + kafka.NewTransportFactory(logger, resolver), ) return &Builder{ diff --git a/config/config.go b/config/config.go index 7ad9b63..add1eff 100644 --- a/config/config.go +++ b/config/config.go @@ -38,6 +38,7 @@ type RetryStrategyConfig struct { type OptionsConfig struct { AutoSetup bool `yaml:"auto_setup" default:"true"` ConsumerPoolSize int `yaml:"consumer_pool_size" default:"10"` + CommitInterval time.Duration `yaml:"commit_interval"` Exchange ExchangeConfig `yaml:"exchange"` Queues map[string]Queue `yaml:"queues"` } diff --git a/examples/kafka_transport/handler/hello_handler.go b/examples/kafka_transport/handler/hello_handler.go new file mode 100644 index 0000000..5488524 --- /dev/null +++ b/examples/kafka_transport/handler/hello_handler.go @@ -0,0 +1,20 @@ +package handler + +import ( + "context" + "fmt" + + "github.com/gerfey/messenger/examples/kafka_transport/message" +) + +type ExampleHelloHandler struct{} + +func (u *ExampleHelloHandler) Handle(_ context.Context, msg *message.ExampleHelloMessage) error { + fmt.Printf("Handled: Text=%v\n", msg.Text) + + return nil +} + +func (u *ExampleHelloHandler) GetBusName() string { + return "default" +} diff --git a/examples/kafka_transport/kafka_transport.go b/examples/kafka_transport/kafka_transport.go new file mode 100644 index 0000000..8d7c075 --- /dev/null +++ b/examples/kafka_transport/kafka_transport.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "log/slog" + + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/examples/kafka_transport/handler" + "github.com/gerfey/messenger/examples/kafka_transport/message" +) + +const ( + waitDurationSeconds = 20 +) + +func main() { + ctx := context.Background() + + log := slog.Default() + + cfg, err := config.LoadConfig("./examples/kafka_transport/messenger.yaml") + if err != nil { + log.Error("ERROR load config", "error", err) + + return + } + + b := builder.NewBuilder(cfg, log) + + _ = b.RegisterHandler(&handler.ExampleHelloHandler{}) + + messenger, err := b.Build() + if err != nil { + log.Error("failed to build messenger", "error", err) + + return + } + + go func() { + if runErr := messenger.Run(ctx); runErr != nil { + log.Error("messenger run failed", "error", runErr) + + return + } + }() + + messengerBus, err := messenger.GetDefaultBus() + if err != nil { + log.Error("failed to get default bus", "error", err) + + return + } + + _, err = messengerBus.Dispatch(ctx, &message.ExampleHelloMessage{ + Text: "Hello World", + }) + if err != nil { + log.Error("failed to dispatch message", "error", err) + + return + } + + <-ctx.Done() +} diff --git a/examples/kafka_transport/message/hello.go b/examples/kafka_transport/message/hello.go new file mode 100644 index 0000000..a7877ee --- /dev/null +++ b/examples/kafka_transport/message/hello.go @@ -0,0 +1,9 @@ +package message + +type ExampleHelloMessage struct { + Text string +} + +func (m *ExampleHelloMessage) RoutingKey() string { + return "test_routing_key" +} diff --git a/examples/kafka_transport/messenger.yaml b/examples/kafka_transport/messenger.yaml new file mode 100644 index 0000000..97d4e3f --- /dev/null +++ b/examples/kafka_transport/messenger.yaml @@ -0,0 +1,19 @@ +default_bus: default + +buses: + default: ~ + +transports: + kafka: + dsn: "kafka://localhost:29092/my-topic?group=my-group&offset=earliest" + retry_strategy: + max_retries: 5 + delay: 500ms + multiplier: 2 + max_delay: 5s + options: + consumer_pool_size: 3 + commit_interval: 500ms + +routing: + message.ExampleHelloMessage: kafka diff --git a/go.mod b/go.mod index 7456130..1733836 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gerfey/messenger -go 1.24.3 +go 1.24 require ( github.com/creasty/defaults v1.8.0 @@ -14,7 +14,10 @@ require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/joho/godotenv v1.5.1 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/segmentio/kafka-go v0.4.48 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index 706d96a..7d99597 100644 --- a/go.sum +++ b/go.sum @@ -2,24 +2,79 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= +github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= diff --git a/transport/amqp/adapters.go b/transport/amqp/adapters.go deleted file mode 100644 index 6f5c25d..0000000 --- a/transport/amqp/adapters.go +++ /dev/null @@ -1,118 +0,0 @@ -package amqp - -import ( - "context" - - amqp "github.com/rabbitmq/amqp091-go" - - "github.com/gerfey/messenger/api" -) - -type ConnectionAdapter struct { - conn *amqp.Connection -} - -func NewConnectionAdapter(conn *amqp.Connection) ConnectionAMQP { - return &ConnectionAdapter{conn: conn} -} - -func (c *ConnectionAdapter) Channel() (ChannelAMQP, error) { - ch, err := c.conn.Channel() - if err != nil { - return nil, err - } - - return NewChannelAdapter(ch), nil -} - -func (c *ConnectionAdapter) IsClosed() bool { - return c.conn.IsClosed() -} - -func (c *ConnectionAdapter) Close() error { - return c.conn.Close() -} - -type ChannelAdapter struct { - ch *amqp.Channel -} - -func NewChannelAdapter(ch *amqp.Channel) ChannelAMQP { - return &ChannelAdapter{ch: ch} -} - -func (c *ChannelAdapter) ExchangeDeclare( - name, kind string, - durable, autoDelete, internal, noWait bool, - args amqp.Table, -) error { - return c.ch.ExchangeDeclare( - name, kind, - durable, autoDelete, internal, noWait, - args, - ) -} - -func (c *ChannelAdapter) QueueDeclare( - name string, - durable, autoDelete, exclusive, noWait bool, - args amqp.Table, -) (amqp.Queue, error) { - return c.ch.QueueDeclare(name, durable, autoDelete, exclusive, noWait, args) -} - -func (c *ChannelAdapter) QueueBind(name, key, exchange string, noWait bool, args amqp.Table) error { - return c.ch.QueueBind(name, key, exchange, noWait, args) -} - -func (c *ChannelAdapter) Publish(exchange, key string, mandatory, immediate bool, msg amqp.Publishing) error { - return c.ch.Publish(exchange, key, mandatory, immediate, msg) -} - -func (c *ChannelAdapter) Consume( - queue, consumer string, - autoAck, exclusive, noLocal, noWait bool, - args amqp.Table, -) (<-chan amqp.Delivery, error) { - return c.ch.Consume(queue, consumer, autoAck, exclusive, noLocal, noWait, args) -} - -func (c *ChannelAdapter) Close() error { - return c.ch.Close() -} - -type PublisherAdapter struct { - publisher *Publisher -} - -func NewPublisherAdapter(publisher *Publisher) PublisherAMQP { - return &PublisherAdapter{publisher: publisher} -} - -func (p *PublisherAdapter) Publish(ctx context.Context, env api.Envelope) error { - return p.publisher.Publish(ctx, env) -} - -type ConsumerAdapter struct { - consumer *Consumer -} - -func NewConsumerAdapter(consumer *Consumer) ConsumerAMQP { - return &ConsumerAdapter{consumer: consumer} -} - -func (c *ConsumerAdapter) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - return c.consumer.Consume(ctx, handler) -} - -type RetryAdapter struct { - retry *Retry -} - -func NewRetryAdapter(retry *Retry) RetryAMQP { - return &RetryAdapter{retry: retry} -} - -func (r *RetryAdapter) Retry(ctx context.Context, env api.Envelope) error { - return r.retry.Retry(ctx, env) -} diff --git a/transport/kafka/config.go b/transport/kafka/config.go new file mode 100644 index 0000000..daf23c9 --- /dev/null +++ b/transport/kafka/config.go @@ -0,0 +1,62 @@ +package kafka + +import ( + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/gerfey/messenger/config" +) + +type TransportConfig struct { + Name string + Brokers []string + Topic string + GroupID string + Offset string + ConsumerPoolSize int + CommitInterval time.Duration +} + +func NewConfig(name, dsn string, opts config.OptionsConfig) (*TransportConfig, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, fmt.Errorf("failed to parse dsn: %w", err) + } + + brokers := strings.Split(u.Host, ",") + topic := strings.TrimPrefix(u.Path, "/") + + query := u.Query() + groupID := query.Get("group") + if groupID == "" { + return nil, errors.New("groupID is required (e.g. ?group=my-group)") + } + + offset := query.Get("offset") + if offset == "" { + offset = "latest" + } + + consumerPoolSize := 1 + if opts.ConsumerPoolSize > 0 { + consumerPoolSize = opts.ConsumerPoolSize + } + + commitInterval := 1 * time.Second + if opts.CommitInterval > 0 { + commitInterval = opts.CommitInterval + } + + return &TransportConfig{ + Name: name, + Brokers: brokers, + Topic: topic, + GroupID: groupID, + Offset: offset, + ConsumerPoolSize: consumerPoolSize, + CommitInterval: commitInterval, + }, nil +} diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go new file mode 100644 index 0000000..7df4d8a --- /dev/null +++ b/transport/kafka/consumer.go @@ -0,0 +1,132 @@ +package kafka + +import ( + "context" + "time" + + "github.com/segmentio/kafka-go" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" +) + +type Consumer struct { + cfg *TransportConfig + serializer api.Serializer +} + +func NewConsumer(cfg *TransportConfig, ser api.Serializer) *Consumer { + return &Consumer{ + cfg: cfg, + serializer: ser, + } +} + +func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + r := kafka.NewReader(kafka.ReaderConfig{ + Brokers: c.cfg.Brokers, + GroupID: c.cfg.GroupID, + Topic: c.cfg.Topic, + StartOffset: c.startOffset(c.cfg.Offset), + CommitInterval: c.cfg.CommitInterval, + MinBytes: 10e3, // 10KB + MaxBytes: 10e6, // 10MB + ReadLagInterval: -1, + + SessionTimeout: 10 * time.Second, + RebalanceTimeout: 5 * time.Second, + HeartbeatInterval: 2 * time.Second, + MaxWait: 1 * time.Second, + }) + defer r.Close() + + jobs := make(chan job) + c.startWorkerPool(ctx, jobs, handler) + + go c.fetchMessages(ctx, r, jobs) + + <-ctx.Done() + + return ctx.Err() +} + +func (c *Consumer) startWorkerPool(ctx context.Context, jobs chan job, handler func(context.Context, api.Envelope) error) { + poolSize := c.cfg.ConsumerPoolSize + if poolSize <= 0 { + poolSize = 10 + } + + for i := 0; i < poolSize; i++ { + go func(workerID int) { + for j := range jobs { + c.handleMessage(ctx, j.r, j.msg, handler) + } + }(i) + } +} + +func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan job) { + for { + select { + case <-ctx.Done(): + close(jobs) + + return + default: + msg, err := r.FetchMessage(ctx) + if err != nil { + if ctx.Err() != nil { + return + } + + continue + } + + jobs <- job{r: r, msg: msg} + } + } +} + +func (c *Consumer) handleMessage( + ctx context.Context, + r *kafka.Reader, + msg kafka.Message, + handler func(context.Context, api.Envelope) error, +) { + env, err := c.serializer.Unmarshal(msg.Value, c.headerMap(msg.Headers)) + if err != nil { + _ = r.CommitMessages(ctx, msg) + + return + } + + env = env.WithStamp(stamps.ReceivedStamp{Transport: c.cfg.Name}) + + if err := handler(ctx, env); err != nil { + _ = r.CommitMessages(ctx, msg) + + return + } + + _ = r.CommitMessages(ctx, msg) +} + +type job struct { + r *kafka.Reader + msg kafka.Message +} + +func (c *Consumer) startOffset(offset string) int64 { + if offset == "earliest" { + return kafka.FirstOffset + } + return kafka.LastOffset +} + +func (c *Consumer) headerMap(hdrs []kafka.Header) map[string]string { + m := make(map[string]string, len(hdrs)) + for _, h := range hdrs { + m[h.Key] = string(h.Value) + } + return m +} diff --git a/transport/kafka/factory.go b/transport/kafka/factory.go new file mode 100644 index 0000000..8acb798 --- /dev/null +++ b/transport/kafka/factory.go @@ -0,0 +1,40 @@ +package kafka + +import ( + "fmt" + "log/slog" + "strings" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/config" +) + +type TransportFactory struct { + logger *slog.Logger + resolver api.TypeResolver +} + +func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { + return &TransportFactory{ + logger: logger, + resolver: resolver, + } +} + +func (t *TransportFactory) Supports(dsn string) bool { + return strings.HasPrefix(dsn, "kafka://") +} + +func (t *TransportFactory) Create(name string, dsn string, options config.OptionsConfig) (api.Transport, error) { + cfg, err := NewConfig(name, dsn, options) + if err != nil { + return nil, fmt.Errorf("create config kafka: %w", err) + } + + transport, err := NewTransport(cfg, t.resolver, t.logger) + if err != nil { + return nil, fmt.Errorf("create kafka transport: %w", err) + } + + return transport, nil +} diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go new file mode 100644 index 0000000..bfc791a --- /dev/null +++ b/transport/kafka/producer.go @@ -0,0 +1,55 @@ +package kafka + +import ( + "context" + "fmt" + "time" + + "github.com/segmentio/kafka-go" + + "github.com/gerfey/messenger/api" +) + +type Producer struct { + writer *kafka.Writer + cfg *TransportConfig + serializer api.Serializer +} + +func NewProducer(cfg *TransportConfig, ser api.Serializer) (*Producer, error) { + return &Producer{ + cfg: cfg, + serializer: ser, + writer: &kafka.Writer{ + Addr: kafka.TCP(cfg.Brokers...), + Topic: cfg.Topic, + RequiredAcks: kafka.RequireAll, + Balancer: &kafka.LeastBytes{}, + Async: false, + }, + }, nil +} + +func (p *Producer) Send(ctx context.Context, env api.Envelope) error { + payload, headers, err := p.serializer.Marshal(env) + if err != nil { + return fmt.Errorf("serializer envelope failed: %w", err) + } + + kHeaders := make([]kafka.Header, 0, len(headers)) + for k, v := range headers { + kHeaders = append(kHeaders, kafka.Header{Key: k, Value: []byte(v)}) + } + + msg := kafka.Message{ + Headers: kHeaders, + Value: payload, + Time: time.Now(), + } + + if err := p.writer.WriteMessages(ctx, msg); err != nil { + return fmt.Errorf("producer failed to write messages: %w", err) + } + + return nil +} diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go new file mode 100644 index 0000000..aaa306b --- /dev/null +++ b/transport/kafka/transport.go @@ -0,0 +1,53 @@ +package kafka + +import ( + "context" + "fmt" + "log/slog" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/serializer" +) + +type Transport struct { + cfg *TransportConfig + producer *Producer + consumer *Consumer + serializer api.Serializer + logger *slog.Logger +} + +func NewTransport(cfg *TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { + ser := serializer.NewSerializer(resolver) + + producer, err := NewProducer(cfg, ser) + if err != nil { + return nil, fmt.Errorf("failed to create kafka producer: %w", err) + } + + consumer := NewConsumer(cfg, ser) + + return &Transport{ + cfg: cfg, + producer: producer, + consumer: consumer, + serializer: ser, + logger: logger, + }, nil +} + +func (t *Transport) Name() string { + return t.cfg.Name +} + +func (t *Transport) Send(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} + +func (t *Transport) Receive(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + return t.consumer.Consume(ctx, handler) +} + +func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} diff --git a/transport/sync/transport.go b/transport/sync/transport.go index e0de523..98994db 100644 --- a/transport/sync/transport.go +++ b/transport/sync/transport.go @@ -17,7 +17,7 @@ func NewTransport(locator api.BusLocator) api.Transport { return &Transport{locator: locator} } -func (t *Transport) Send(_ context.Context, env api.Envelope) error { +func (t *Transport) Send(ctx context.Context, env api.Envelope) error { busNameStump, ok := envelope.LastStampOf[stamps.BusNameStamp](env) if !ok { return errors.New("no BusNameStamp found in envelope") @@ -30,7 +30,7 @@ func (t *Transport) Send(_ context.Context, env api.Envelope) error { env = env.WithStamp(stamps.ReceivedStamp{Transport: t.Name()}) - _, err := messageBus.Dispatch(context.Background(), env) + _, err := messageBus.Dispatch(ctx, env) if err != nil { return err } From e8c60f1cbc68f8b1c1f24d1d605f2ad212ab2923 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Tue, 29 Jul 2025 20:00:29 +0700 Subject: [PATCH 02/17] refactor: simplify the configuration of transports and remove dependence on the general configuration --- api/transport.go | 4 +- builder/builder.go | 3 +- config/config.go | 25 +- examples/config/config.go | 20 +- examples/kafka_transport/kafka_transport.go | 4 - examples/kafka_transport/messenger.yaml | 16 +- go.mod | 4 +- go.sum | 5 + tests/helpers/messages.go | 3 +- tests/mocks/mock_amqp.go | 311 -------------------- tests/mocks/mock_transport.go | 3 +- transport/amqp/config.go | 27 +- transport/amqp/factory.go | 18 +- transport/amqp/factory_test.go | 7 +- transport/amqp/interfaces.go | 42 --- transport/amqp/transport_test.go | 271 ----------------- transport/chain.go | 9 +- transport/inmemory/config.go | 6 +- transport/inmemory/factory.go | 11 +- transport/inmemory/transport.go | 10 +- transport/inmemory/transport_test.go | 44 +-- transport/kafka/config.go | 62 +--- transport/kafka/consumer.go | 22 +- transport/kafka/factory.go | 31 +- transport/kafka/producer.go | 8 +- transport/kafka/transport.go | 4 +- transport/sync/factory.go | 3 +- 27 files changed, 167 insertions(+), 806 deletions(-) delete mode 100644 tests/mocks/mock_amqp.go delete mode 100644 transport/amqp/interfaces.go delete mode 100644 transport/amqp/transport_test.go diff --git a/api/transport.go b/api/transport.go index 7087681..d99806a 100644 --- a/api/transport.go +++ b/api/transport.go @@ -3,8 +3,6 @@ package api import ( "context" "reflect" - - "github.com/gerfey/messenger/config" ) type Transport interface { @@ -35,7 +33,7 @@ type SenderLocator interface { type TransportFactory interface { Supports(string) bool - Create(string, string, config.OptionsConfig) (Transport, error) + Create(string, string, []byte) (Transport, error) } type RoutedMessage interface { diff --git a/builder/builder.go b/builder/builder.go index cea9488..49332c7 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -275,8 +275,7 @@ func (b *Builder) registerStamps() { func (b *Builder) createdSyncTransport(createdTransports map[string]api.Transport) { cfg := config.TransportConfig{ - DSN: "sync://", - Options: config.OptionsConfig{}, + DSN: "sync://", } if syncTransport, err := b.transportFactory.CreateTransport("sync", cfg); err == nil { diff --git a/config/config.go b/config/config.go index add1eff..3093c35 100644 --- a/config/config.go +++ b/config/config.go @@ -25,7 +25,7 @@ type BusConfig struct { type TransportConfig struct { DSN string `yaml:"dsn"` RetryStrategy *RetryStrategyConfig `yaml:"retry_strategy"` - Options OptionsConfig `yaml:"options"` + Options map[string]any `yaml:"options"` } type RetryStrategyConfig struct { @@ -35,29 +35,6 @@ type RetryStrategyConfig struct { MaxDelay time.Duration `yaml:"max_delay"` } -type OptionsConfig struct { - AutoSetup bool `yaml:"auto_setup" default:"true"` - ConsumerPoolSize int `yaml:"consumer_pool_size" default:"10"` - CommitInterval time.Duration `yaml:"commit_interval"` - Exchange ExchangeConfig `yaml:"exchange"` - Queues map[string]Queue `yaml:"queues"` -} - -type ExchangeConfig struct { - Name string `yaml:"name"` - Type string `yaml:"type" default:"topic"` // topic, direct, fanout - Durable bool `yaml:"durable" default:"true"` - AutoDelete bool `yaml:"auto_delete" default:"false"` - Internal bool `yaml:"internal" default:"false"` -} - -type Queue struct { - BindingKeys []string `yaml:"binding_keys"` - Durable bool `yaml:"durable" default:"true"` - Exclusive bool `yaml:"exclusive" default:"false"` - AutoDelete bool `yaml:"auto_delete" default:"false"` -} - type SerializedEnvelope struct { Message any `json:"message"` MessageType string `json:"type"` diff --git a/examples/config/config.go b/examples/config/config.go index 96e4137..8cbc630 100644 --- a/examples/config/config.go +++ b/examples/config/config.go @@ -6,7 +6,10 @@ import ( "log/slog" "os" + "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/transport/amqp" ) func main() { @@ -39,13 +42,18 @@ func main() { fmt.Printf(" MaxDelay: %v\n", transport.RetryStrategy.MaxDelay) } + rawYAML, _ := yaml.Marshal(transport.Options) + + var opts amqp.OptionsConfig + _ = yaml.Unmarshal(rawYAML, &opts) + fmt.Printf(" Options:\n") - fmt.Printf(" AutoSetup: %v\n", transport.Options.AutoSetup) + fmt.Printf(" AutoSetup: %v\n", opts.AutoSetup) fmt.Printf(" Exchange:\n") - fmt.Printf(" Name: %s\n", transport.Options.Exchange.Name) - fmt.Printf(" Type: %s\n", transport.Options.Exchange.Type) - fmt.Printf(" Durable: %v\n", transport.Options.Exchange.Durable) - fmt.Printf(" AutoDelete: %v\n", transport.Options.Exchange.AutoDelete) - fmt.Printf(" Internal: %v\n", transport.Options.Exchange.Internal) + fmt.Printf(" Name: %s\n", opts.Exchange.Name) + fmt.Printf(" Type: %s\n", opts.Exchange.Type) + fmt.Printf(" Durable: %v\n", opts.Exchange.Durable) + fmt.Printf(" AutoDelete: %v\n", opts.Exchange.AutoDelete) + fmt.Printf(" Internal: %v\n", opts.Exchange.Internal) } } diff --git a/examples/kafka_transport/kafka_transport.go b/examples/kafka_transport/kafka_transport.go index 8d7c075..5c73da9 100644 --- a/examples/kafka_transport/kafka_transport.go +++ b/examples/kafka_transport/kafka_transport.go @@ -10,10 +10,6 @@ import ( "github.com/gerfey/messenger/examples/kafka_transport/message" ) -const ( - waitDurationSeconds = 20 -) - func main() { ctx := context.Background() diff --git a/examples/kafka_transport/messenger.yaml b/examples/kafka_transport/messenger.yaml index 97d4e3f..6774874 100644 --- a/examples/kafka_transport/messenger.yaml +++ b/examples/kafka_transport/messenger.yaml @@ -1,19 +1,33 @@ default_bus: default +failure_transport: failed_messages buses: default: ~ transports: kafka: - dsn: "kafka://localhost:29092/my-topic?group=my-group&offset=earliest" + dsn: "kafka://localhost:29092/" retry_strategy: max_retries: 5 delay: 500ms multiplier: 2 max_delay: 5s options: + topic: my-topic + group: my-group + offset: earliest consumer_pool_size: 3 commit_interval: 500ms + failed_messages: + dsn: "amqp://guest:guest@localhost:5672/" + options: + auto_setup: true + exchange: + name: failed_exchange + type: fanout + queues: + failed_messages_queue: ~ + routing: message.ExampleHelloMessage: kafka diff --git a/go.mod b/go.mod index 1733836..dbc6aac 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,10 @@ require ( github.com/creasty/defaults v1.8.0 github.com/ilyakaznacheev/cleanenv v1.5.0 github.com/rabbitmq/amqp091-go v1.10.0 + github.com/segmentio/kafka-go v0.4.48 github.com/stretchr/testify v1.10.0 go.uber.org/mock v0.5.2 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -17,7 +19,5 @@ require ( github.com/klauspost/compress v1.15.9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/segmentio/kafka-go v0.4.48 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index 7d99597..96b7164 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,11 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -43,6 +46,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -66,6 +70,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/tests/helpers/messages.go b/tests/helpers/messages.go index 76244ee..2efaaf1 100644 --- a/tests/helpers/messages.go +++ b/tests/helpers/messages.go @@ -5,7 +5,6 @@ import ( "errors" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type TestMessage struct { @@ -221,7 +220,7 @@ func (f *TestTransportFactory) Supports(_ string) bool { return true } -func (f *TestTransportFactory) Create(_ string, _ string, _ config.OptionsConfig) (api.Transport, error) { +func (f *TestTransportFactory) Create(_ string, _ string, _ []byte) (api.Transport, error) { if f.CreateError != nil { return nil, f.CreateError } diff --git a/tests/mocks/mock_amqp.go b/tests/mocks/mock_amqp.go deleted file mode 100644 index cac474c..0000000 --- a/tests/mocks/mock_amqp.go +++ /dev/null @@ -1,311 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: interfaces.go -// -// Generated by this command: -// -// mockgen -source=interfaces.go -destination=../../tests/mocks/mock_amqp.go -package=mocks -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - api "github.com/gerfey/messenger/api" - amqp "github.com/gerfey/messenger/transport/amqp" - amqp091 "github.com/rabbitmq/amqp091-go" - gomock "go.uber.org/mock/gomock" -) - -// MockConnectionAMQP is a mock of ConnectionAMQP interface. -type MockConnectionAMQP struct { - ctrl *gomock.Controller - recorder *MockConnectionAMQPMockRecorder - isgomock struct{} -} - -// MockConnectionAMQPMockRecorder is the mock recorder for MockConnectionAMQP. -type MockConnectionAMQPMockRecorder struct { - mock *MockConnectionAMQP -} - -// NewMockConnectionAMQP creates a new mock instance. -func NewMockConnectionAMQP(ctrl *gomock.Controller) *MockConnectionAMQP { - mock := &MockConnectionAMQP{ctrl: ctrl} - mock.recorder = &MockConnectionAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConnectionAMQP) EXPECT() *MockConnectionAMQPMockRecorder { - return m.recorder -} - -// Channel mocks base method. -func (m *MockConnectionAMQP) Channel() (amqp.ChannelAMQP, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Channel") - ret0, _ := ret[0].(amqp.ChannelAMQP) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Channel indicates an expected call of Channel. -func (mr *MockConnectionAMQPMockRecorder) Channel() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Channel", reflect.TypeOf((*MockConnectionAMQP)(nil).Channel)) -} - -// Close mocks base method. -func (m *MockConnectionAMQP) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockConnectionAMQPMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockConnectionAMQP)(nil).Close)) -} - -// IsClosed mocks base method. -func (m *MockConnectionAMQP) IsClosed() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsClosed") - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsClosed indicates an expected call of IsClosed. -func (mr *MockConnectionAMQPMockRecorder) IsClosed() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsClosed", reflect.TypeOf((*MockConnectionAMQP)(nil).IsClosed)) -} - -// MockChannelAMQP is a mock of ChannelAMQP interface. -type MockChannelAMQP struct { - ctrl *gomock.Controller - recorder *MockChannelAMQPMockRecorder - isgomock struct{} -} - -// MockChannelAMQPMockRecorder is the mock recorder for MockChannelAMQP. -type MockChannelAMQPMockRecorder struct { - mock *MockChannelAMQP -} - -// NewMockChannelAMQP creates a new mock instance. -func NewMockChannelAMQP(ctrl *gomock.Controller) *MockChannelAMQP { - mock := &MockChannelAMQP{ctrl: ctrl} - mock.recorder = &MockChannelAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockChannelAMQP) EXPECT() *MockChannelAMQPMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockChannelAMQP) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockChannelAMQPMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockChannelAMQP)(nil).Close)) -} - -// Consume mocks base method. -func (m *MockChannelAMQP) Consume(queue, consumer string, autoAck, exclusive, noLocal, noWait bool, args amqp091.Table) (<-chan amqp091.Delivery, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Consume", queue, consumer, autoAck, exclusive, noLocal, noWait, args) - ret0, _ := ret[0].(<-chan amqp091.Delivery) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Consume indicates an expected call of Consume. -func (mr *MockChannelAMQPMockRecorder) Consume(queue, consumer, autoAck, exclusive, noLocal, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Consume", reflect.TypeOf((*MockChannelAMQP)(nil).Consume), queue, consumer, autoAck, exclusive, noLocal, noWait, args) -} - -// ExchangeDeclare mocks base method. -func (m *MockChannelAMQP) ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp091.Table) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExchangeDeclare", name, kind, durable, autoDelete, internal, noWait, args) - ret0, _ := ret[0].(error) - return ret0 -} - -// ExchangeDeclare indicates an expected call of ExchangeDeclare. -func (mr *MockChannelAMQPMockRecorder) ExchangeDeclare(name, kind, durable, autoDelete, internal, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExchangeDeclare", reflect.TypeOf((*MockChannelAMQP)(nil).ExchangeDeclare), name, kind, durable, autoDelete, internal, noWait, args) -} - -// Publish mocks base method. -func (m *MockChannelAMQP) Publish(exchange, key string, mandatory, immediate bool, msg amqp091.Publishing) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Publish", exchange, key, mandatory, immediate, msg) - ret0, _ := ret[0].(error) - return ret0 -} - -// Publish indicates an expected call of Publish. -func (mr *MockChannelAMQPMockRecorder) Publish(exchange, key, mandatory, immediate, msg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockChannelAMQP)(nil).Publish), exchange, key, mandatory, immediate, msg) -} - -// QueueBind mocks base method. -func (m *MockChannelAMQP) QueueBind(name, key, exchange string, noWait bool, args amqp091.Table) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "QueueBind", name, key, exchange, noWait, args) - ret0, _ := ret[0].(error) - return ret0 -} - -// QueueBind indicates an expected call of QueueBind. -func (mr *MockChannelAMQPMockRecorder) QueueBind(name, key, exchange, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueBind", reflect.TypeOf((*MockChannelAMQP)(nil).QueueBind), name, key, exchange, noWait, args) -} - -// QueueDeclare mocks base method. -func (m *MockChannelAMQP) QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp091.Table) (amqp091.Queue, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "QueueDeclare", name, durable, autoDelete, exclusive, noWait, args) - ret0, _ := ret[0].(amqp091.Queue) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// QueueDeclare indicates an expected call of QueueDeclare. -func (mr *MockChannelAMQPMockRecorder) QueueDeclare(name, durable, autoDelete, exclusive, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueDeclare", reflect.TypeOf((*MockChannelAMQP)(nil).QueueDeclare), name, durable, autoDelete, exclusive, noWait, args) -} - -// MockPublisherAMQP is a mock of PublisherAMQP interface. -type MockPublisherAMQP struct { - ctrl *gomock.Controller - recorder *MockPublisherAMQPMockRecorder - isgomock struct{} -} - -// MockPublisherAMQPMockRecorder is the mock recorder for MockPublisherAMQP. -type MockPublisherAMQPMockRecorder struct { - mock *MockPublisherAMQP -} - -// NewMockPublisherAMQP creates a new mock instance. -func NewMockPublisherAMQP(ctrl *gomock.Controller) *MockPublisherAMQP { - mock := &MockPublisherAMQP{ctrl: ctrl} - mock.recorder = &MockPublisherAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPublisherAMQP) EXPECT() *MockPublisherAMQPMockRecorder { - return m.recorder -} - -// Publish mocks base method. -func (m *MockPublisherAMQP) Publish(ctx context.Context, env api.Envelope) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Publish", ctx, env) - ret0, _ := ret[0].(error) - return ret0 -} - -// Publish indicates an expected call of Publish. -func (mr *MockPublisherAMQPMockRecorder) Publish(ctx, env any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockPublisherAMQP)(nil).Publish), ctx, env) -} - -// MockConsumerAMQP is a mock of ConsumerAMQP interface. -type MockConsumerAMQP struct { - ctrl *gomock.Controller - recorder *MockConsumerAMQPMockRecorder - isgomock struct{} -} - -// MockConsumerAMQPMockRecorder is the mock recorder for MockConsumerAMQP. -type MockConsumerAMQPMockRecorder struct { - mock *MockConsumerAMQP -} - -// NewMockConsumerAMQP creates a new mock instance. -func NewMockConsumerAMQP(ctrl *gomock.Controller) *MockConsumerAMQP { - mock := &MockConsumerAMQP{ctrl: ctrl} - mock.recorder = &MockConsumerAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConsumerAMQP) EXPECT() *MockConsumerAMQPMockRecorder { - return m.recorder -} - -// Consume mocks base method. -func (m *MockConsumerAMQP) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Consume", ctx, handler) - ret0, _ := ret[0].(error) - return ret0 -} - -// Consume indicates an expected call of Consume. -func (mr *MockConsumerAMQPMockRecorder) Consume(ctx, handler any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Consume", reflect.TypeOf((*MockConsumerAMQP)(nil).Consume), ctx, handler) -} - -// MockRetryAMQP is a mock of RetryAMQP interface. -type MockRetryAMQP struct { - ctrl *gomock.Controller - recorder *MockRetryAMQPMockRecorder - isgomock struct{} -} - -// MockRetryAMQPMockRecorder is the mock recorder for MockRetryAMQP. -type MockRetryAMQPMockRecorder struct { - mock *MockRetryAMQP -} - -// NewMockRetryAMQP creates a new mock instance. -func NewMockRetryAMQP(ctrl *gomock.Controller) *MockRetryAMQP { - mock := &MockRetryAMQP{ctrl: ctrl} - mock.recorder = &MockRetryAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRetryAMQP) EXPECT() *MockRetryAMQPMockRecorder { - return m.recorder -} - -// Retry mocks base method. -func (m *MockRetryAMQP) Retry(ctx context.Context, env api.Envelope) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Retry", ctx, env) - ret0, _ := ret[0].(error) - return ret0 -} - -// Retry indicates an expected call of Retry. -func (mr *MockRetryAMQPMockRecorder) Retry(ctx, env any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Retry", reflect.TypeOf((*MockRetryAMQP)(nil).Retry), ctx, env) -} diff --git a/tests/mocks/mock_transport.go b/tests/mocks/mock_transport.go index 83fdd92..3d69a78 100644 --- a/tests/mocks/mock_transport.go +++ b/tests/mocks/mock_transport.go @@ -14,7 +14,6 @@ import ( reflect "reflect" api "github.com/gerfey/messenger/api" - config "github.com/gerfey/messenger/config" gomock "go.uber.org/mock/gomock" ) @@ -355,7 +354,7 @@ func (m *MockTransportFactory) EXPECT() *MockTransportFactoryMockRecorder { } // Create mocks base method. -func (m *MockTransportFactory) Create(arg0, arg1 string, arg2 config.OptionsConfig) (api.Transport, error) { +func (m *MockTransportFactory) Create(arg0, arg1 string, arg2 []byte) (api.Transport, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) ret0, _ := ret[0].(api.Transport) diff --git a/transport/amqp/config.go b/transport/amqp/config.go index 955c163..391aa60 100644 --- a/transport/amqp/config.go +++ b/transport/amqp/config.go @@ -1,11 +1,34 @@ package amqp import ( - "github.com/gerfey/messenger/config" + "time" ) type TransportConfig struct { Name string DSN string - Options config.OptionsConfig + Options OptionsConfig +} + +type OptionsConfig struct { + AutoSetup bool `yaml:"auto_setup" default:"false"` + ConsumerPoolSize int `yaml:"consumer_pool_size" default:"10"` + CommitInterval time.Duration `yaml:"commit_interval" default:"10"` + Exchange ExchangeConfig `yaml:"exchange"` + Queues map[string]Queue `yaml:"queues"` +} + +type ExchangeConfig struct { + Name string `yaml:"name"` + Type string `yaml:"type" default:"topic"` // topic, direct, fanout + Durable bool `yaml:"durable" default:"true"` + AutoDelete bool `yaml:"auto_delete" default:"false"` + Internal bool `yaml:"internal" default:"false"` +} + +type Queue struct { + BindingKeys []string `yaml:"binding_keys"` + Durable bool `yaml:"durable" default:"true"` + Exclusive bool `yaml:"exclusive" default:"false"` + AutoDelete bool `yaml:"auto_delete" default:"false"` } diff --git a/transport/amqp/factory.go b/transport/amqp/factory.go index c0c2513..65dbd6d 100644 --- a/transport/amqp/factory.go +++ b/transport/amqp/factory.go @@ -1,11 +1,14 @@ package amqp import ( + "fmt" "log/slog" "strings" + "github.com/creasty/defaults" + "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type TransportFactory struct { @@ -24,11 +27,20 @@ func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "amqp://") } -func (f *TransportFactory) Create(name string, dsn string, options config.OptionsConfig) (api.Transport, error) { +func (f *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { + var opts OptionsConfig + if err := defaults.Set(&opts); err != nil { + return nil, fmt.Errorf("amqp: set defaults: %w", err) + } + + if err := yaml.Unmarshal(options, &opts); err != nil { + return nil, fmt.Errorf("amqp: unmarshal options: %w", err) + } + cfg := TransportConfig{ Name: name, DSN: dsn, - Options: options, + Options: opts, } return NewTransport(cfg, f.resolver, f.logger) diff --git a/transport/amqp/factory_test.go b/transport/amqp/factory_test.go index 59f128a..de56c4e 100644 --- a/transport/amqp/factory_test.go +++ b/transport/amqp/factory_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/amqp" ) @@ -85,16 +84,16 @@ func TestTransportFactory_Create(t *testing.T) { name := "test-amqp" dsn := "amqp://guest:guest@non-existent-host:5672/" - options := config.OptionsConfig{ + options := amqp.OptionsConfig{ AutoSetup: true, - Exchange: config.ExchangeConfig{ + Exchange: amqp.ExchangeConfig{ Name: "test-exchange", Type: "direct", Durable: true, AutoDelete: false, Internal: false, }, - Queues: map[string]config.Queue{ + Queues: map[string]amqp.Queue{ "test-queue": { BindingKeys: []string{"test-key"}, Durable: true, diff --git a/transport/amqp/interfaces.go b/transport/amqp/interfaces.go deleted file mode 100644 index 29bc927..0000000 --- a/transport/amqp/interfaces.go +++ /dev/null @@ -1,42 +0,0 @@ -package amqp - -import ( - "context" - - amqp "github.com/rabbitmq/amqp091-go" - - "github.com/gerfey/messenger/api" -) - -//go:generate go run go.uber.org/mock/mockgen@latest -source=interfaces.go -destination=../../tests/mocks/mock_amqp.go -package=mocks - -type ConnectionAMQP interface { - Channel() (ChannelAMQP, error) - IsClosed() bool - Close() error -} - -type ChannelAMQP interface { - ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp.Table) error - QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp.Table) (amqp.Queue, error) - QueueBind(name, key, exchange string, noWait bool, args amqp.Table) error - Publish(exchange, key string, mandatory, immediate bool, msg amqp.Publishing) error - Consume( - queue, consumer string, - autoAck, exclusive, noLocal, noWait bool, - args amqp.Table, - ) (<-chan amqp.Delivery, error) - Close() error -} - -type PublisherAMQP interface { - Publish(ctx context.Context, env api.Envelope) error -} - -type ConsumerAMQP interface { - Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error -} - -type RetryAMQP interface { - Retry(ctx context.Context, env api.Envelope) error -} diff --git a/transport/amqp/transport_test.go b/transport/amqp/transport_test.go deleted file mode 100644 index 1201a64..0000000 --- a/transport/amqp/transport_test.go +++ /dev/null @@ -1,271 +0,0 @@ -package amqp_test - -import ( - "context" - "errors" - "testing" - "time" - - amqp091 "github.com/rabbitmq/amqp091-go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - - "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" - "github.com/gerfey/messenger/core/envelope" - "github.com/gerfey/messenger/tests/helpers" - "github.com/gerfey/messenger/tests/mocks" - "github.com/gerfey/messenger/transport/amqp" -) - -func TestTransport_Name(t *testing.T) { - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: false, - }, - } - - resolver := builder.NewResolver() - logger, _ := helpers.NewFakeLogger() - - transport, err := amqp.NewTransport(cfg, resolver, logger) - if err != nil { - t.Skip("Skipping test - requires AMQP connection") - - return - } - - assert.Equal(t, "test-transport", transport.Name()) -} - -func TestTransport_Setup_WithMocks(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - t.Run("successful setup", func(t *testing.T) { - mockConn := mocks.NewMockConnectionAMQP(ctrl) - mockChannel := mocks.NewMockChannelAMQP(ctrl) - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: true, - Exchange: config.ExchangeConfig{ - Name: "test-exchange", - Type: "direct", - Durable: true, - AutoDelete: false, - Internal: false, - }, - Queues: map[string]config.Queue{ - "test-queue": { - Durable: true, - AutoDelete: false, - Exclusive: false, - BindingKeys: []string{"test.key"}, - }, - }, - }, - } - - mockConn.EXPECT().Channel().Return(mockChannel, nil) - mockChannel.EXPECT().ExchangeDeclare( - "test-exchange", "direct", true, false, false, false, nil, - ).Return(nil) - mockChannel.EXPECT().QueueDeclare( - "test-queue", true, false, false, false, nil, - ).Return(amqp091.Queue{Name: "test-queue"}, nil) - mockChannel.EXPECT().QueueBind( - "test-queue", "test.key", "test-exchange", false, nil, - ).Return(nil) - mockChannel.EXPECT().Close().Return(nil) - - transport := createMockTransport(cfg, mockConn) - err := transport.Setup() - require.NoError(t, err) - }) - - t.Run("setup with channel error", func(t *testing.T) { - mockConn := mocks.NewMockConnectionAMQP(ctrl) - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: true, - }, - } - - expectedErr := errors.New("channel error") - mockConn.EXPECT().Channel().Return(nil, expectedErr) - - transport := createMockTransport(cfg, mockConn) - err := transport.Setup() - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to open channel") - }) - - t.Run("setup with exchange declare error", func(t *testing.T) { - mockConn := mocks.NewMockConnectionAMQP(ctrl) - mockChannel := mocks.NewMockChannelAMQP(ctrl) - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: true, - Exchange: config.ExchangeConfig{ - Name: "test-exchange", - Type: "direct", - }, - }, - } - - expectedErr := errors.New("exchange error") - mockConn.EXPECT().Channel().Return(mockChannel, nil) - mockChannel.EXPECT().ExchangeDeclare( - gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(expectedErr) - mockChannel.EXPECT().Close().Return(nil) - - transport := createMockTransport(cfg, mockConn) - err := transport.Setup() - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to declare exchange") - }) -} - -func TestTransport_Integration_Methods(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: false, - }, - } - - resolver := builder.NewResolver() - resolver.RegisterMessage(&helpers.TestMessage{}) - logger, _ := helpers.NewFakeLogger() - - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() - - transport, err := amqp.NewTransport(cfg, resolver, logger) - if err != nil { - t.Skip("Skipping integration test - requires AMQP connection:", err) - - return - } - - msg := &helpers.TestMessage{ID: "123", Content: "test message"} - env := envelope.NewEnvelope(msg) - - t.Run("send method delegates to publisher", func(t *testing.T) { - sendCtx, sendCancel := context.WithTimeout(ctx, 2*time.Second) - defer sendCancel() - - sendErr := transport.Send(sendCtx, env) - assert.NoError(t, sendErr) - }) - - t.Run("receive method delegates to consumer", func(t *testing.T) { - handler := func(_ context.Context, _ api.Envelope) error { - return nil - } - - receiveCtx, receiveCancel := context.WithTimeout(ctx, 2*time.Second) - defer receiveCancel() - - receiveErr := transport.Receive(receiveCtx, handler) - assert.Error(t, receiveErr) - }) - - t.Run("retry method delegates to retry", func(t *testing.T) { - retryableTransport, ok := transport.(api.RetryableTransport) - if !ok { - t.Skip("Transport does not implement RetryableTransport") - - return - } - - retryCtx, retryCancel := context.WithTimeout(ctx, 2*time.Second) - defer retryCancel() - - retryErr := retryableTransport.Retry(retryCtx, env) - - assert.NoError(t, retryErr) - }) -} - -type MockTransport struct { - cfg amqp.TransportConfig - conn *mocks.MockConnectionAMQP -} - -func createMockTransport(cfg amqp.TransportConfig, conn *mocks.MockConnectionAMQP) *MockTransport { - return &MockTransport{ - cfg: cfg, - conn: conn, - } -} - -func (t *MockTransport) Setup() error { - ch, err := t.conn.Channel() - if err != nil { - return errors.New("failed to open channel: " + err.Error()) - } - defer ch.Close() - - err = ch.ExchangeDeclare( - t.cfg.Options.Exchange.Name, - t.cfg.Options.Exchange.Type, - t.cfg.Options.Exchange.Durable, - t.cfg.Options.Exchange.AutoDelete, - t.cfg.Options.Exchange.Internal, - false, - nil, - ) - if err != nil { - return errors.New("failed to declare exchange: " + err.Error()) - } - - for queueName, queueCfg := range t.cfg.Options.Queues { - _, err = ch.QueueDeclare( - queueName, - queueCfg.Durable, - queueCfg.AutoDelete, - queueCfg.Exclusive, - false, - nil, - ) - if err != nil { - return errors.New("declare queue: " + err.Error()) - } - - for _, bindingKey := range queueCfg.BindingKeys { - bindErr := ch.QueueBind( - queueName, - bindingKey, - t.cfg.Options.Exchange.Name, - false, - nil, - ) - if bindErr != nil { - return errors.New("bind queue: " + bindErr.Error()) - } - } - } - - return nil -} diff --git a/transport/chain.go b/transport/chain.go index 0726b9f..5fbaf74 100644 --- a/transport/chain.go +++ b/transport/chain.go @@ -3,6 +3,8 @@ package transport import ( "fmt" + "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/api" "github.com/gerfey/messenger/config" ) @@ -18,7 +20,12 @@ func NewFactoryChain(factories ...api.TransportFactory) *FactoryChain { func (c *FactoryChain) CreateTransport(name string, config config.TransportConfig) (api.Transport, error) { for _, factory := range c.factories { if factory.Supports(config.DSN) { - return factory.Create(name, config.DSN, config.Options) + rawOptions, errOptions := yaml.Marshal(config.Options) + if errOptions != nil { + return nil, fmt.Errorf("%s: marshal options map: %w", name, errOptions) + } + + return factory.Create(name, config.DSN, rawOptions) } } diff --git a/transport/inmemory/config.go b/transport/inmemory/config.go index a48147d..4d373f3 100644 --- a/transport/inmemory/config.go +++ b/transport/inmemory/config.go @@ -1,11 +1,7 @@ package inmemory -import ( - "github.com/gerfey/messenger/config" -) - type TransportConfig struct { Name string DSN string - Options config.OptionsConfig + Options any } diff --git a/transport/inmemory/factory.go b/transport/inmemory/factory.go index 600a227..3dd680d 100644 --- a/transport/inmemory/factory.go +++ b/transport/inmemory/factory.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type TransportFactory struct { @@ -24,12 +23,6 @@ func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "in-memory://") } -func (f *TransportFactory) Create(name string, dsn string, options config.OptionsConfig) (api.Transport, error) { - cfg := TransportConfig{ - Name: name, - DSN: dsn, - Options: options, - } - - return NewTransport(cfg), nil +func (f *TransportFactory) Create(name string, _ string, _ []byte) (api.Transport, error) { + return NewTransport(name), nil } diff --git a/transport/inmemory/transport.go b/transport/inmemory/transport.go index 861e2fe..404f910 100644 --- a/transport/inmemory/transport.go +++ b/transport/inmemory/transport.go @@ -12,20 +12,20 @@ import ( const sleepDuration = 10 * time.Millisecond type Transport struct { - cfg TransportConfig + name string queue []api.Envelope lock sync.Mutex } -func NewTransport(cfg TransportConfig) api.Transport { +func NewTransport(name string) api.Transport { return &Transport{ - cfg: cfg, + name: name, queue: make([]api.Envelope, 0), } } func (t *Transport) Name() string { - return t.cfg.Name + return t.name } func (t *Transport) Send(_ context.Context, env api.Envelope) error { @@ -56,7 +56,7 @@ func (t *Transport) Receive(ctx context.Context, handler func(context.Context, a t.queue = t.queue[1:] t.lock.Unlock() - envWithReceivedStamp := env.WithStamp(stamps.ReceivedStamp{Transport: t.cfg.Name}) + envWithReceivedStamp := env.WithStamp(stamps.ReceivedStamp{Transport: t.name}) if err := handler(ctx, envWithReceivedStamp); err != nil { return err diff --git a/transport/inmemory/transport_test.go b/transport/inmemory/transport_test.go index e1cf2f8..1b8b677 100644 --- a/transport/inmemory/transport_test.go +++ b/transport/inmemory/transport_test.go @@ -19,9 +19,7 @@ import ( func TestNewTransport(t *testing.T) { t.Run("create transport with config", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") require.NotNil(t, transport) assert.IsType(t, &inmemory.Transport{}, transport) @@ -29,9 +27,7 @@ func TestNewTransport(t *testing.T) { }) t.Run("create transport with empty config", func(t *testing.T) { - cfg := inmemory.TransportConfig{} - - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("in-memory") require.NotNil(t, transport) assert.Empty(t, transport.Name()) @@ -40,16 +36,14 @@ func TestNewTransport(t *testing.T) { func TestTransport_Name(t *testing.T) { t.Run("get transport name", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "my-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("my-transport") name := transport.Name() assert.Equal(t, "my-transport", name) }) t.Run("get empty transport name", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: ""} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("") name := transport.Name() assert.Empty(t, name) @@ -58,8 +52,7 @@ func TestTransport_Name(t *testing.T) { func TestTransport_Send(t *testing.T) { t.Run("send single message", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg).(*inmemory.Transport) + transport := inmemory.NewTransport("test-transport").(*inmemory.Transport) msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -70,8 +63,7 @@ func TestTransport_Send(t *testing.T) { }) t.Run("send multiple messages", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg).(*inmemory.Transport) + transport := inmemory.NewTransport("test-transport").(*inmemory.Transport) msg1 := &helpers.TestMessage{Content: "test1"} msg2 := &helpers.TestMessage{Content: "test2"} @@ -91,8 +83,7 @@ func TestTransport_Send(t *testing.T) { }) t.Run("send with cancelled context", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -107,8 +98,7 @@ func TestTransport_Send(t *testing.T) { func TestTransport_Receive(t *testing.T) { t.Run("receive single message", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -139,8 +129,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive multiple messages", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg1 := &helpers.TestMessage{Content: "test1"} msg2 := &helpers.TestMessage{Content: "test2"} @@ -170,8 +159,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive with handler error", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -192,8 +180,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive with cancelled context", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") ctx, cancel := context.WithCancel(t.Context()) cancel() @@ -208,8 +195,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive with empty queue waits", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") handler := func(_ context.Context, _ api.Envelope) error { return nil @@ -229,8 +215,7 @@ func TestTransport_Receive(t *testing.T) { func TestTransport_Integration(t *testing.T) { t.Run("full send and receive workflow", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "integration-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("integration-transport") messages := []*helpers.TestMessage{ {Content: "message1"}, @@ -269,8 +254,7 @@ func TestTransport_Integration(t *testing.T) { }) t.Run("concurrent send and receive", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "concurrent-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("concurrent-transport") go func() { for range 5 { diff --git a/transport/kafka/config.go b/transport/kafka/config.go index daf23c9..367bd37 100644 --- a/transport/kafka/config.go +++ b/transport/kafka/config.go @@ -1,62 +1,20 @@ package kafka import ( - "errors" - "fmt" - "net/url" - "strings" "time" - - "github.com/gerfey/messenger/config" ) type TransportConfig struct { - Name string - Brokers []string - Topic string - GroupID string - Offset string - ConsumerPoolSize int - CommitInterval time.Duration + Name string + DSN string + Options OptionsConfig } -func NewConfig(name, dsn string, opts config.OptionsConfig) (*TransportConfig, error) { - u, err := url.Parse(dsn) - if err != nil { - return nil, fmt.Errorf("failed to parse dsn: %w", err) - } - - brokers := strings.Split(u.Host, ",") - topic := strings.TrimPrefix(u.Path, "/") - - query := u.Query() - groupID := query.Get("group") - if groupID == "" { - return nil, errors.New("groupID is required (e.g. ?group=my-group)") - } - - offset := query.Get("offset") - if offset == "" { - offset = "latest" - } - - consumerPoolSize := 1 - if opts.ConsumerPoolSize > 0 { - consumerPoolSize = opts.ConsumerPoolSize - } - - commitInterval := 1 * time.Second - if opts.CommitInterval > 0 { - commitInterval = opts.CommitInterval - } - - return &TransportConfig{ - Name: name, - Brokers: brokers, - Topic: topic, - GroupID: groupID, - Offset: offset, - ConsumerPoolSize: consumerPoolSize, - CommitInterval: commitInterval, - }, nil +type OptionsConfig struct { + ConsumerPoolSize int `yaml:"consumer_pool_size" default:"10"` + CommitInterval time.Duration `yaml:"commit_interval"` + Offset string `yaml:"offset" default:"latest"` + Group string `yaml:"group" default:"group"` + Topic string `yaml:"topic" default:"topic"` + Brokers []string } diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index 7df4d8a..dea85c3 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -11,11 +11,11 @@ import ( ) type Consumer struct { - cfg *TransportConfig + cfg TransportConfig serializer api.Serializer } -func NewConsumer(cfg *TransportConfig, ser api.Serializer) *Consumer { +func NewConsumer(cfg TransportConfig, ser api.Serializer) *Consumer { return &Consumer{ cfg: cfg, serializer: ser, @@ -24,11 +24,11 @@ func NewConsumer(cfg *TransportConfig, ser api.Serializer) *Consumer { func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { r := kafka.NewReader(kafka.ReaderConfig{ - Brokers: c.cfg.Brokers, - GroupID: c.cfg.GroupID, - Topic: c.cfg.Topic, - StartOffset: c.startOffset(c.cfg.Offset), - CommitInterval: c.cfg.CommitInterval, + Brokers: c.cfg.Options.Brokers, + GroupID: c.cfg.Options.Group, + Topic: c.cfg.Options.Topic, + StartOffset: c.startOffset(c.cfg.Options.Offset), + CommitInterval: c.cfg.Options.CommitInterval, MinBytes: 10e3, // 10KB MaxBytes: 10e6, // 10MB ReadLagInterval: -1, @@ -50,8 +50,12 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap return ctx.Err() } -func (c *Consumer) startWorkerPool(ctx context.Context, jobs chan job, handler func(context.Context, api.Envelope) error) { - poolSize := c.cfg.ConsumerPoolSize +func (c *Consumer) startWorkerPool( + ctx context.Context, + jobs chan job, + handler func(context.Context, api.Envelope) error, +) { + poolSize := c.cfg.Options.ConsumerPoolSize if poolSize <= 0 { poolSize = 10 } diff --git a/transport/kafka/factory.go b/transport/kafka/factory.go index 8acb798..44ad0d0 100644 --- a/transport/kafka/factory.go +++ b/transport/kafka/factory.go @@ -3,10 +3,13 @@ package kafka import ( "fmt" "log/slog" + "net/url" "strings" + "github.com/creasty/defaults" + "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type TransportFactory struct { @@ -25,16 +28,28 @@ func (t *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "kafka://") } -func (t *TransportFactory) Create(name string, dsn string, options config.OptionsConfig) (api.Transport, error) { - cfg, err := NewConfig(name, dsn, options) - if err != nil { - return nil, fmt.Errorf("create config kafka: %w", err) +func (t *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { + var optsConfig OptionsConfig + if err := defaults.Set(&optsConfig); err != nil { + return nil, fmt.Errorf("kafka: set defaults: %w", err) + } + + if err := yaml.Unmarshal(options, &optsConfig); err != nil { + return nil, fmt.Errorf("kafka: unmarshal options: %w", err) } - transport, err := NewTransport(cfg, t.resolver, t.logger) + u, err := url.Parse(dsn) if err != nil { - return nil, fmt.Errorf("create kafka transport: %w", err) + return nil, fmt.Errorf("kafka: failed to parse dsn: %w", err) + } + + optsConfig.Brokers = strings.Split(u.Host, ",") + + tCfg := TransportConfig{ + Name: name, + DSN: dsn, + Options: optsConfig, } - return transport, nil + return NewTransport(tCfg, t.resolver, t.logger) } diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index bfc791a..bf8533e 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -12,17 +12,17 @@ import ( type Producer struct { writer *kafka.Writer - cfg *TransportConfig + cfg TransportConfig serializer api.Serializer } -func NewProducer(cfg *TransportConfig, ser api.Serializer) (*Producer, error) { +func NewProducer(cfg TransportConfig, ser api.Serializer) (*Producer, error) { return &Producer{ cfg: cfg, serializer: ser, writer: &kafka.Writer{ - Addr: kafka.TCP(cfg.Brokers...), - Topic: cfg.Topic, + Addr: kafka.TCP(cfg.Options.Brokers...), + Topic: cfg.Options.Topic, RequiredAcks: kafka.RequireAll, Balancer: &kafka.LeastBytes{}, Async: false, diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index aaa306b..1d094c9 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -10,14 +10,14 @@ import ( ) type Transport struct { - cfg *TransportConfig + cfg TransportConfig producer *Producer consumer *Consumer serializer api.Serializer logger *slog.Logger } -func NewTransport(cfg *TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { +func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { ser := serializer.NewSerializer(resolver) producer, err := NewProducer(cfg, ser) diff --git a/transport/sync/factory.go b/transport/sync/factory.go index 822e4df..63e8a59 100644 --- a/transport/sync/factory.go +++ b/transport/sync/factory.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type Factory struct { @@ -24,6 +23,6 @@ func (f *Factory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "sync://") } -func (f *Factory) Create(_ string, _ string, _ config.OptionsConfig) (api.Transport, error) { +func (f *Factory) Create(_ string, _ string, _ []byte) (api.Transport, error) { return NewTransport(f.locator), nil } From d5a0fe2afe529a5efbaaff8d84b4b785d6d6e0cf Mon Sep 17 00:00:00 2001 From: Gerfey Date: Tue, 29 Jul 2025 21:00:16 +0700 Subject: [PATCH 03/17] refactor: fix tests and code style --- builder/builder_test.go | 36 +++++++------- config/config_test.go | 74 +++++++++++++++++++++++----- config/parser_test.go | 59 ++++++++++++++++++---- transport/amqp/factory_test.go | 6 ++- transport/chain_test.go | 14 +++--- transport/inmemory/factory_test.go | 9 ++-- transport/inmemory/transport_test.go | 2 +- transport/kafka/config.go | 2 +- transport/kafka/consumer.go | 42 ++++++++++------ transport/kafka/producer.go | 4 +- transport/sync/factory_test.go | 9 ++-- 11 files changed, 185 insertions(+), 72 deletions(-) diff --git a/builder/builder_test.go b/builder/builder_test.go index 57913d0..8ea6516 100644 --- a/builder/builder_test.go +++ b/builder/builder_test.go @@ -25,9 +25,9 @@ func TestNewBuilder(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -210,9 +210,9 @@ func TestBuilder_RegisterTransportFactory(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -265,9 +265,9 @@ func TestBuilder_Build_Errors(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -296,9 +296,9 @@ func TestBuilder_Build_Errors(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -331,9 +331,9 @@ func TestBuilder_Build_Success(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -377,9 +377,9 @@ func TestBuilder_Build_Success(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, diff --git a/config/config_test.go b/config/config_test.go index 9d8532a..ffb0c69 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -54,7 +54,7 @@ func TestLoadConfig(t *testing.T) { require.NoError(t, err) assert.Equal(t, "memory://test-host", cfg.Transports["default"].DSN) - assert.Equal(t, 15, cfg.Transports["default"].Options.ConsumerPoolSize) + assert.Equal(t, 15, cfg.Transports["default"].Options["consumer_pool_size"]) assert.Equal(t, "amqp://guest:guest@localhost:5672/", cfg.Transports["amqp"].DSN) assert.Equal(t, uint(5), cfg.Transports["amqp"].RetryStrategy.MaxRetries) }) @@ -121,8 +121,14 @@ transports: require.NoError(t, err) assert.Equal(t, "memory://default", cfg.Transports["default"].DSN) - assert.True(t, cfg.Transports["default"].Options.AutoSetup) - assert.Equal(t, 10, cfg.Transports["default"].Options.ConsumerPoolSize) + autoSetup, ok := cfg.Transports["default"].Options["auto_setup"] + if ok && autoSetup != nil { + assert.True(t, autoSetup.(bool)) + } + consumerPoolSize, ok := cfg.Transports["default"].Options["consumer_pool_size"] + if ok && consumerPoolSize != nil { + assert.Equal(t, 10, consumerPoolSize) + } }) t.Run("exchange options default values", func(t *testing.T) { @@ -139,11 +145,33 @@ transports: `), &cfg) require.NoError(t, err) - assert.Equal(t, "messages", cfg.Transports["amqp"].Options.Exchange.Name) - assert.Equal(t, "topic", cfg.Transports["amqp"].Options.Exchange.Type) - assert.True(t, cfg.Transports["amqp"].Options.Exchange.Durable) - assert.False(t, cfg.Transports["amqp"].Options.Exchange.AutoDelete) - assert.False(t, cfg.Transports["amqp"].Options.Exchange.Internal) + exchangeOptions, ok := cfg.Transports["amqp"].Options["exchange"].(map[string]any) + require.True(t, ok) + + name, ok := exchangeOptions["name"] + if ok && name != nil { + assert.Equal(t, "messages", name) + } + + exchangeType, ok := exchangeOptions["type"] + if ok && exchangeType != nil { + assert.Equal(t, "topic", exchangeType) + } + + durable, ok := exchangeOptions["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } + + autoDelete, ok := exchangeOptions["auto_delete"] + if ok && autoDelete != nil { + assert.False(t, autoDelete.(bool)) + } + + internal, ok := exchangeOptions["internal"] + if ok && internal != nil { + assert.False(t, internal.(bool)) + } }) t.Run("queue options default values", func(t *testing.T) { @@ -162,9 +190,31 @@ transports: `), &cfg) require.NoError(t, err) - assert.Contains(t, cfg.Transports["amqp"].Options.Queues["default"].BindingKeys, "#") - assert.True(t, cfg.Transports["amqp"].Options.Queues["default"].Durable) - assert.False(t, cfg.Transports["amqp"].Options.Queues["default"].Exclusive) - assert.False(t, cfg.Transports["amqp"].Options.Queues["default"].AutoDelete) + + queuesOptions, ok := cfg.Transports["amqp"].Options["queues"].(map[string]any) + require.True(t, ok) + + defaultQueue, ok := queuesOptions["default"].(map[string]any) + require.True(t, ok) + + bindingKeys, ok := defaultQueue["binding_keys"] + if ok && bindingKeys != nil { + assert.Contains(t, bindingKeys, "#") + } + + durable, ok := defaultQueue["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } + + exclusive, ok := defaultQueue["exclusive"] + if ok && exclusive != nil { + assert.False(t, exclusive.(bool)) + } + + autoDelete, ok := defaultQueue["auto_delete"] + if ok && autoDelete != nil { + assert.False(t, autoDelete.(bool)) + } }) } diff --git a/config/parser_test.go b/config/parser_test.go index 6d72117..5e53130 100644 --- a/config/parser_test.go +++ b/config/parser_test.go @@ -42,8 +42,16 @@ transports: require.NoError(t, err) assert.Equal(t, "custom", cfg.DefaultBus) - assert.True(t, cfg.Transports["default"].Options.AutoSetup) - assert.Equal(t, 10, cfg.Transports["default"].Options.ConsumerPoolSize) + + autoSetup, ok := cfg.Transports["default"].Options["auto_setup"] + if ok && autoSetup != nil { + assert.True(t, autoSetup.(bool)) + } + + consumerPoolSize, ok := cfg.Transports["default"].Options["consumer_pool_size"] + if ok && consumerPoolSize != nil { + assert.Equal(t, 10, consumerPoolSize) + } }) t.Run("parse invalid yaml", func(t *testing.T) { @@ -86,10 +94,29 @@ transports: err := parser.Parse(content, &cfg) require.NoError(t, err) - assert.Equal(t, "messages", cfg.Transports["amqp"].Options.Exchange.Name) - assert.Equal(t, "topic", cfg.Transports["amqp"].Options.Exchange.Type) - assert.True(t, cfg.Transports["amqp"].Options.Exchange.Durable) - assert.False(t, cfg.Transports["amqp"].Options.Exchange.AutoDelete) + + exchangeOptions, ok := cfg.Transports["amqp"].Options["exchange"].(map[string]interface{}) + require.True(t, ok) + + name, ok := exchangeOptions["name"] + if ok && name != nil { + assert.Equal(t, "messages", name) + } + + exchangeType, ok := exchangeOptions["type"] + if ok && exchangeType != nil { + assert.Equal(t, "topic", exchangeType) + } + + durable, ok := exchangeOptions["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } + + autoDelete, ok := exchangeOptions["auto_delete"] + if ok && autoDelete != nil { + assert.False(t, autoDelete.(bool)) + } }) t.Run("parse yaml with queue options", func(t *testing.T) { @@ -107,9 +134,23 @@ transports: err := parser.Parse(content, &cfg) require.NoError(t, err) - assert.Contains(t, cfg.Transports["amqp"].Options.Queues, "default") - assert.Contains(t, cfg.Transports["amqp"].Options.Queues["default"].BindingKeys, "#") - assert.True(t, cfg.Transports["amqp"].Options.Queues["default"].Durable) + + queuesOptions, ok := cfg.Transports["amqp"].Options["queues"].(map[string]interface{}) + require.True(t, ok) + assert.Contains(t, queuesOptions, "default") + + defaultQueue, ok := queuesOptions["default"].(map[string]interface{}) + require.True(t, ok) + + bindingKeys, ok := defaultQueue["binding_keys"].([]interface{}) + if ok && bindingKeys != nil { + assert.Contains(t, bindingKeys, "#") + } + + durable, ok := defaultQueue["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } }) t.Run("parse yaml with retry strategy", func(t *testing.T) { diff --git a/transport/amqp/factory_test.go b/transport/amqp/factory_test.go index de56c4e..7f65377 100644 --- a/transport/amqp/factory_test.go +++ b/transport/amqp/factory_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/amqp" @@ -103,7 +104,10 @@ func TestTransportFactory_Create(t *testing.T) { }, } - _, err := factory.Create(name, dsn, options) + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) + + _, err = factory.Create(name, dsn, optionsBytes) require.Error(t, err) assert.Contains(t, err.Error(), "failed to connect") diff --git a/transport/chain_test.go b/transport/chain_test.go index 7677188..6c72ead 100644 --- a/transport/chain_test.go +++ b/transport/chain_test.go @@ -23,7 +23,7 @@ func (f *mockFactory) Supports(dsn string) bool { return dsn == f.supportedDSN } -func (f *mockFactory) Create(name, _ string, _ config.OptionsConfig) (api.Transport, error) { +func (f *mockFactory) Create(name, _ string, _ []byte) (api.Transport, error) { if f.createError != nil { return nil, f.createError } @@ -81,7 +81,7 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } tr, err := chain.CreateTransport("test-transport", transportConfig) @@ -103,7 +103,7 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } tr, err := chain.CreateTransport("first-transport", transportConfig) @@ -119,7 +119,7 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "unknown://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } tr, err := chain.CreateTransport("unknown-transport", transportConfig) @@ -141,7 +141,7 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } tr, err := chain.CreateTransport("error-transport", transportConfig) @@ -156,7 +156,7 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } tr, err := chain.CreateTransport("test-transport", transportConfig) @@ -179,7 +179,7 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "redis://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } tr, err := chain.CreateTransport("second-transport", transportConfig) diff --git a/transport/inmemory/factory_test.go b/transport/inmemory/factory_test.go index 3c1db29..661ad88 100644 --- a/transport/inmemory/factory_test.go +++ b/transport/inmemory/factory_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/inmemory" ) @@ -79,9 +79,12 @@ func TestTransportFactory_Create(t *testing.T) { name := "test-inmemory" dsn := "in-memory://test" - options := config.OptionsConfig{} + options := map[string]any{} - transport, err := factory.Create(name, dsn, options) + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) + + transport, err := factory.Create(name, dsn, optionsBytes) require.NoError(t, err) assert.NotNil(t, transport) diff --git a/transport/inmemory/transport_test.go b/transport/inmemory/transport_test.go index 1b8b677..e7e5522 100644 --- a/transport/inmemory/transport_test.go +++ b/transport/inmemory/transport_test.go @@ -30,7 +30,7 @@ func TestNewTransport(t *testing.T) { transport := inmemory.NewTransport("in-memory") require.NotNil(t, transport) - assert.Empty(t, transport.Name()) + assert.Equal(t, "in-memory", transport.Name()) }) } diff --git a/transport/kafka/config.go b/transport/kafka/config.go index 367bd37..f8dc2b7 100644 --- a/transport/kafka/config.go +++ b/transport/kafka/config.go @@ -16,5 +16,5 @@ type OptionsConfig struct { Offset string `yaml:"offset" default:"latest"` Group string `yaml:"group" default:"group"` Topic string `yaml:"topic" default:"topic"` - Brokers []string + Brokers []string `yaml:"brokers"` } diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index dea85c3..b493b05 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -10,6 +10,16 @@ import ( "github.com/gerfey/messenger/core/stamps" ) +const ( + minBytes = 10e3 // 10KB + maxBytes = 10e6 // 10MB + sessionTimeout = 10 * time.Second + rebalanceTimeout = 5 * time.Second + heartbeatInterval = 2 * time.Second + defaultPoolSize = 10 + readLagInterval = -1 +) + type Consumer struct { cfg TransportConfig serializer api.Serializer @@ -29,14 +39,14 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap Topic: c.cfg.Options.Topic, StartOffset: c.startOffset(c.cfg.Options.Offset), CommitInterval: c.cfg.Options.CommitInterval, - MinBytes: 10e3, // 10KB - MaxBytes: 10e6, // 10MB - ReadLagInterval: -1, - - SessionTimeout: 10 * time.Second, - RebalanceTimeout: 5 * time.Second, - HeartbeatInterval: 2 * time.Second, - MaxWait: 1 * time.Second, + MinBytes: minBytes, + MaxBytes: maxBytes, + ReadLagInterval: readLagInterval, + + SessionTimeout: sessionTimeout, + RebalanceTimeout: rebalanceTimeout, + HeartbeatInterval: heartbeatInterval, + MaxWait: time.Second, }) defer r.Close() @@ -57,11 +67,11 @@ func (c *Consumer) startWorkerPool( ) { poolSize := c.cfg.Options.ConsumerPoolSize if poolSize <= 0 { - poolSize = 10 + poolSize = defaultPoolSize } - for i := 0; i < poolSize; i++ { - go func(workerID int) { + for i := range make([]struct{}, poolSize) { + go func(_ int) { for j := range jobs { c.handleMessage(ctx, j.r, j.msg, handler) } @@ -106,7 +116,7 @@ func (c *Consumer) handleMessage( env = env.WithStamp(stamps.ReceivedStamp{Transport: c.cfg.Name}) - if err := handler(ctx, env); err != nil { + if handlerErr := handler(ctx, env); handlerErr != nil { _ = r.CommitMessages(ctx, msg) return @@ -124,13 +134,15 @@ func (c *Consumer) startOffset(offset string) int64 { if offset == "earliest" { return kafka.FirstOffset } + return kafka.LastOffset } -func (c *Consumer) headerMap(hdrs []kafka.Header) map[string]string { - m := make(map[string]string, len(hdrs)) - for _, h := range hdrs { +func (c *Consumer) headerMap(headers []kafka.Header) map[string]string { + m := make(map[string]string, len(headers)) + for _, h := range headers { m[h.Key] = string(h.Value) } + return m } diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index bf8533e..0e36deb 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -47,8 +47,8 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { Time: time.Now(), } - if err := p.writer.WriteMessages(ctx, msg); err != nil { - return fmt.Errorf("producer failed to write messages: %w", err) + if writeErr := p.writer.WriteMessages(ctx, msg); writeErr != nil { + return fmt.Errorf("producer failed to write messages: %w", writeErr) } return nil diff --git a/transport/sync/factory_test.go b/transport/sync/factory_test.go index 9e1e6f7..4f0b400 100644 --- a/transport/sync/factory_test.go +++ b/transport/sync/factory_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/sync" ) @@ -79,9 +79,12 @@ func TestFactory_Create(t *testing.T) { name := "test-sync" dsn := "sync://" - options := config.OptionsConfig{} + options := map[string]any{} - transport, err := factory.Create(name, dsn, options) + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) + + transport, err := factory.Create(name, dsn, optionsBytes) require.NoError(t, err) assert.NotNil(t, transport) From 5d0b0f8542d10034b9b6ad7d579533d2bab445c6 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Tue, 29 Jul 2025 21:10:57 +0700 Subject: [PATCH 04/17] test: add tests factory kafka --- transport/kafka/factory_test.go | 103 ++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 transport/kafka/factory_test.go diff --git a/transport/kafka/factory_test.go b/transport/kafka/factory_test.go new file mode 100644 index 0000000..f3852d0 --- /dev/null +++ b/transport/kafka/factory_test.go @@ -0,0 +1,103 @@ +package kafka_test + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/tests/mocks" + "github.com/gerfey/messenger/transport/kafka" +) + +func TestNewTransportFactory(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + logger := slog.Default() + mockResolver := mocks.NewMockTypeResolver(ctrl) + + factory := kafka.NewTransportFactory(logger, mockResolver) + + assert.NotNil(t, factory) + assert.IsType(t, &kafka.TransportFactory{}, factory) +} + +func TestTransportFactory_Supports(t *testing.T) { + testCases := []struct { + name string + dsn string + expected bool + }{ + { + name: "supports kafka dsn", + dsn: "kafka://localhost:9092", + expected: true, + }, + { + name: "does not support amqp dsn", + dsn: "amqp://guest:guest@localhost:5672/", + expected: false, + }, + { + name: "does not support in-memory dsn", + dsn: "in-memory://", + expected: false, + }, + { + name: "does not support sync dsn", + dsn: "sync://", + expected: false, + }, + { + name: "does not support empty dsn", + dsn: "", + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + logger := slog.Default() + mockResolver := mocks.NewMockTypeResolver(ctrl) + factory := kafka.NewTransportFactory(logger, mockResolver) + + result := factory.Supports(tc.dsn) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestTransportFactory_Create(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + logger := slog.Default() + mockResolver := mocks.NewMockTypeResolver(ctrl) + factory := kafka.NewTransportFactory(logger, mockResolver) + + name := "test-kafka" + dsn := "kafka://non-existent-host:9092" + options := kafka.OptionsConfig{ + ConsumerPoolSize: 5, + Offset: "earliest", + Group: "test-group", + Topic: "test-topic", + CommitInterval: 1000000000, + } + + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) + + transport, err := factory.Create(name, dsn, optionsBytes) + + require.NoError(t, err) + assert.NotNil(t, transport) + assert.IsType(t, &kafka.Transport{}, transport) +} From 02944bf4463a69466da8a59f22e0cd9e5336a9af Mon Sep 17 00:00:00 2001 From: Gerfey Date: Tue, 29 Jul 2025 21:45:58 +0700 Subject: [PATCH 05/17] feat: added connection class to manage connections to Kafka --- examples/kafka_transport/kafka_transport.go | 7 ++- transport/kafka/connection.go | 69 +++++++++++++++++++++ transport/kafka/consumer.go | 11 ++-- transport/kafka/factory_test.go | 6 +- transport/kafka/producer.go | 10 +-- transport/kafka/transport.go | 13 +++- 6 files changed, 98 insertions(+), 18 deletions(-) create mode 100644 transport/kafka/connection.go diff --git a/examples/kafka_transport/kafka_transport.go b/examples/kafka_transport/kafka_transport.go index 5c73da9..d7aedc8 100644 --- a/examples/kafka_transport/kafka_transport.go +++ b/examples/kafka_transport/kafka_transport.go @@ -3,6 +3,7 @@ package main import ( "context" "log/slog" + "time" "github.com/gerfey/messenger/builder" "github.com/gerfey/messenger/config" @@ -10,6 +11,10 @@ import ( "github.com/gerfey/messenger/examples/kafka_transport/message" ) +const ( + waitDurationSeconds = 20 +) + func main() { ctx := context.Background() @@ -57,5 +62,5 @@ func main() { return } - <-ctx.Done() + time.Sleep(waitDurationSeconds * time.Second) } diff --git a/transport/kafka/connection.go b/transport/kafka/connection.go new file mode 100644 index 0000000..08c9d61 --- /dev/null +++ b/transport/kafka/connection.go @@ -0,0 +1,69 @@ +package kafka + +import ( + "context" + "fmt" + "time" + + "github.com/segmentio/kafka-go" +) + +const ( + connectionTimeout = 5 * time.Second +) + +type Connection struct { + brokers []string + dialer *kafka.Dialer +} + +func NewConnection(brokers []string) (*Connection, error) { + conn := &Connection{ + brokers: brokers, + dialer: &kafka.Dialer{ + Timeout: connectionTimeout, + DualStack: true, + }, + } + + if err := conn.Check(connectionTimeout); err != nil { + return nil, err + } + + return conn, nil +} + +func (c *Connection) Check(timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + for _, broker := range c.brokers { + conn, connErr := c.dialer.DialContext(ctx, "tcp", broker) + if connErr != nil { + return fmt.Errorf("failed to connect to Kafka broker at '%s': %w", broker, connErr) + } + + if closeErr := conn.Close(); closeErr != nil { + return fmt.Errorf("failed to close connection to Kafka broker at '%s': %w", broker, closeErr) + } + } + + return nil +} + +func (c *Connection) CreateReader(config kafka.ReaderConfig) *kafka.Reader { + config.Brokers = c.brokers + config.Dialer = c.dialer + + return kafka.NewReader(config) +} + +func (c *Connection) CreateWriter(topic string) *kafka.Writer { + return &kafka.Writer{ + Addr: kafka.TCP(c.brokers...), + Topic: topic, + RequiredAcks: kafka.RequireAll, + Balancer: &kafka.LeastBytes{}, + Async: false, + } +} diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index b493b05..8b5121f 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -23,18 +23,19 @@ const ( type Consumer struct { cfg TransportConfig serializer api.Serializer + conn *Connection } -func NewConsumer(cfg TransportConfig, ser api.Serializer) *Consumer { +func NewConsumer(cfg TransportConfig, ser api.Serializer, conn *Connection) *Consumer { return &Consumer{ cfg: cfg, serializer: ser, + conn: conn, } } func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - r := kafka.NewReader(kafka.ReaderConfig{ - Brokers: c.cfg.Options.Brokers, + readerConfig := kafka.ReaderConfig{ GroupID: c.cfg.Options.Group, Topic: c.cfg.Options.Topic, StartOffset: c.startOffset(c.cfg.Options.Offset), @@ -47,7 +48,9 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap RebalanceTimeout: rebalanceTimeout, HeartbeatInterval: heartbeatInterval, MaxWait: time.Second, - }) + } + + r := c.conn.CreateReader(readerConfig) defer r.Close() jobs := make(chan job) diff --git a/transport/kafka/factory_test.go b/transport/kafka/factory_test.go index f3852d0..147d0aa 100644 --- a/transport/kafka/factory_test.go +++ b/transport/kafka/factory_test.go @@ -97,7 +97,7 @@ func TestTransportFactory_Create(t *testing.T) { transport, err := factory.Create(name, dsn, optionsBytes) - require.NoError(t, err) - assert.NotNil(t, transport) - assert.IsType(t, &kafka.Transport{}, transport) + require.Error(t, err) + assert.Contains(t, err.Error(), "kafka") + assert.Nil(t, transport) } diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index 0e36deb..b57a76f 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -16,17 +16,11 @@ type Producer struct { serializer api.Serializer } -func NewProducer(cfg TransportConfig, ser api.Serializer) (*Producer, error) { +func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection) (*Producer, error) { return &Producer{ cfg: cfg, serializer: ser, - writer: &kafka.Writer{ - Addr: kafka.TCP(cfg.Options.Brokers...), - Topic: cfg.Options.Topic, - RequiredAcks: kafka.RequireAll, - Balancer: &kafka.LeastBytes{}, - Async: false, - }, + writer: conn.CreateWriter(cfg.Options.Topic), }, nil } diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index 1d094c9..1b753df 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -15,17 +15,25 @@ type Transport struct { consumer *Consumer serializer api.Serializer logger *slog.Logger + conn *Connection } func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { + conn, err := NewConnection(cfg.Options.Brokers) + if err != nil { + logger.Error("failed to connect to Kafka brokers", "error", err) + + return nil, fmt.Errorf("kafka: %w", err) + } + ser := serializer.NewSerializer(resolver) - producer, err := NewProducer(cfg, ser) + producer, err := NewProducer(cfg, ser, conn) if err != nil { return nil, fmt.Errorf("failed to create kafka producer: %w", err) } - consumer := NewConsumer(cfg, ser) + consumer := NewConsumer(cfg, ser, conn) return &Transport{ cfg: cfg, @@ -33,6 +41,7 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L consumer: consumer, serializer: ser, logger: logger, + conn: conn, }, nil } From ba7f73914a0d0d443271cb01dc3e0062dca7a758 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Tue, 29 Jul 2025 21:54:40 +0700 Subject: [PATCH 06/17] refactor: moving DSN parsing from factory to transport for kafka --- transport/kafka/config.go | 1 - transport/kafka/factory.go | 8 -------- transport/kafka/transport.go | 11 ++++++++++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/transport/kafka/config.go b/transport/kafka/config.go index f8dc2b7..801e4d9 100644 --- a/transport/kafka/config.go +++ b/transport/kafka/config.go @@ -16,5 +16,4 @@ type OptionsConfig struct { Offset string `yaml:"offset" default:"latest"` Group string `yaml:"group" default:"group"` Topic string `yaml:"topic" default:"topic"` - Brokers []string `yaml:"brokers"` } diff --git a/transport/kafka/factory.go b/transport/kafka/factory.go index 44ad0d0..a2bc117 100644 --- a/transport/kafka/factory.go +++ b/transport/kafka/factory.go @@ -3,7 +3,6 @@ package kafka import ( "fmt" "log/slog" - "net/url" "strings" "github.com/creasty/defaults" @@ -38,13 +37,6 @@ func (t *TransportFactory) Create(name string, dsn string, options []byte) (api. return nil, fmt.Errorf("kafka: unmarshal options: %w", err) } - u, err := url.Parse(dsn) - if err != nil { - return nil, fmt.Errorf("kafka: failed to parse dsn: %w", err) - } - - optsConfig.Brokers = strings.Split(u.Host, ",") - tCfg := TransportConfig{ Name: name, DSN: dsn, diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index 1b753df..ebd0ce4 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log/slog" + "net/url" + "strings" "github.com/gerfey/messenger/api" "github.com/gerfey/messenger/serializer" @@ -19,7 +21,14 @@ type Transport struct { } func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { - conn, err := NewConnection(cfg.Options.Brokers) + u, err := url.Parse(cfg.DSN) + if err != nil { + return nil, fmt.Errorf("kafka: failed to parse dsn: %w", err) + } + + brokers := strings.Split(u.Host, ",") + + conn, err := NewConnection(brokers) if err != nil { logger.Error("failed to connect to Kafka brokers", "error", err) From 7f6a65c58b65f0ae411b936f9210ba78f103d295 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Sat, 2 Aug 2025 23:38:43 +0700 Subject: [PATCH 07/17] feat: improved Kafka configuration --- builder/builder.go | 3 + .../implementation/add_message_id.go | 32 ++ core/stamps/message_stamps.go | 5 + examples/kafka_transport/messenger.yaml | 21 +- go.mod | 1 + go.sum | 2 + transport/kafka/config.go | 42 ++- transport/kafka/consumer.go | 296 +++++++++++++++--- transport/kafka/factory_test.go | 7 +- transport/kafka/producer.go | 60 +++- transport/kafka/transport.go | 4 +- 11 files changed, 410 insertions(+), 63 deletions(-) create mode 100644 core/middleware/implementation/add_message_id.go create mode 100644 core/stamps/message_stamps.go diff --git a/builder/builder.go b/builder/builder.go index 49332c7..2a9a89a 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -119,6 +119,8 @@ func (b *Builder) setupBuses() error { } chain = append(chain, implementation.NewAddBusNameMiddleware(name)) + chain = append(chain, implementation.NewAddMessageIDMiddleware()) + chain = append( chain, implementation.NewSendMessageMiddleware(b.logger, b.senderLocator, b.eventDispatcher), @@ -271,6 +273,7 @@ func (b *Builder) setupRetryListeners(createdTransports map[string]api.Transport func (b *Builder) registerStamps() { b.resolver.RegisterStamp(stamps.BusNameStamp{}) b.resolver.RegisterStamp(stamps.RedeliveryStamp{}) + b.resolver.RegisterStamp(stamps.MessageIDStamp{}) } func (b *Builder) createdSyncTransport(createdTransports map[string]api.Transport) { diff --git a/core/middleware/implementation/add_message_id.go b/core/middleware/implementation/add_message_id.go new file mode 100644 index 0000000..76e2b68 --- /dev/null +++ b/core/middleware/implementation/add_message_id.go @@ -0,0 +1,32 @@ +package implementation + +import ( + "context" + + "github.com/google/uuid" + + "github.com/gerfey/messenger/core/envelope" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" +) + +type AddMessageIDMiddleware struct{} + +func NewAddMessageIDMiddleware() *AddMessageIDMiddleware { + return &AddMessageIDMiddleware{} +} + +func (m *AddMessageIDMiddleware) Handle( + ctx context.Context, + env api.Envelope, + next api.NextFunc, +) (api.Envelope, error) { + if _, ok := envelope.LastStampOf[stamps.MessageIDStamp](env); !ok { + env = env.WithStamp(stamps.MessageIDStamp{ + MessageID: uuid.New().String(), + }) + } + + return next(ctx, env) +} diff --git a/core/stamps/message_stamps.go b/core/stamps/message_stamps.go new file mode 100644 index 0000000..7412aa0 --- /dev/null +++ b/core/stamps/message_stamps.go @@ -0,0 +1,5 @@ +package stamps + +type MessageIDStamp struct { + MessageID string +} diff --git a/examples/kafka_transport/messenger.yaml b/examples/kafka_transport/messenger.yaml index 6774874..db0e13d 100644 --- a/examples/kafka_transport/messenger.yaml +++ b/examples/kafka_transport/messenger.yaml @@ -13,11 +13,24 @@ transports: multiplier: 2 max_delay: 5s options: - topic: my-topic + topics: + - my-topic group: my-group - offset: earliest - consumer_pool_size: 3 - commit_interval: 500ms + offset_config: + type: earliest + commit: + strategy: batch + interval: 500ms + batch_size: 10 + pool: + size: 3 + min_size: 2 + max_size: 10 + dynamic: true + rebalance: + strategy: roundrobin + key: + strategy: message_id failed_messages: dsn: "amqp://guest:guest@localhost:5672/" diff --git a/go.mod b/go.mod index dbc6aac..c47cb2a 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect diff --git a/go.sum b/go.sum index 96b7164..5b63c18 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbD github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= diff --git a/transport/kafka/config.go b/transport/kafka/config.go index 801e4d9..c18971a 100644 --- a/transport/kafka/config.go +++ b/transport/kafka/config.go @@ -1,8 +1,6 @@ package kafka -import ( - "time" -) +import "time" type TransportConfig struct { Name string @@ -11,9 +9,37 @@ type TransportConfig struct { } type OptionsConfig struct { - ConsumerPoolSize int `yaml:"consumer_pool_size" default:"10"` - CommitInterval time.Duration `yaml:"commit_interval"` - Offset string `yaml:"offset" default:"latest"` - Group string `yaml:"group" default:"group"` - Topic string `yaml:"topic" default:"topic"` + Topics []string `yaml:"topics,omitempty"` + Group string `yaml:"group" default:"group"` + OffsetConfig OffsetConfig `yaml:"offset_config"` + Commit CommitConfig `yaml:"commit"` + Pool PoolConfig `yaml:"pool"` + Rebalance RebalanceConfig `yaml:"rebalance"` + Key KeyConfig `yaml:"key"` +} + +type OffsetConfig struct { + Type string `yaml:"type" default:"latest"` // earliest, latest, specific + Value int64 `yaml:"value"` +} + +type CommitConfig struct { + Strategy string `yaml:"strategy" default:"auto"` + Interval time.Duration `yaml:"interval" default:"1s"` + BatchSize int `yaml:"batch_size" default:"100"` +} + +type PoolConfig struct { + Size int `yaml:"size" default:"10"` + MinSize int `yaml:"min_size" default:"5"` + MaxSize int `yaml:"max_size" default:"50"` + Dynamic bool `yaml:"dynamic" default:"false"` +} + +type RebalanceConfig struct { + Strategy string `yaml:"strategy" default:"range"` // range, roundrobin +} + +type KeyConfig struct { + Strategy string `yaml:"strategy" default:"none"` // none, message_id } diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index 8b5121f..dcd4e0d 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -2,6 +2,9 @@ package kafka import ( "context" + "fmt" + "log/slog" + "sync" "time" "github.com/segmentio/kafka-go" @@ -21,73 +24,195 @@ const ( ) type Consumer struct { - cfg TransportConfig - serializer api.Serializer - conn *Connection + cfg TransportConfig + serializer api.Serializer + conn *Connection + readers []*kafka.Reader + wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc + batchMutex sync.Mutex + batchMessages []kafka.Message + deferredCommits sync.Map + logger *slog.Logger } -func NewConsumer(cfg TransportConfig, ser api.Serializer, conn *Connection) *Consumer { +type messageWithReader struct { + message kafka.Message + reader *kafka.Reader +} + +func NewConsumer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) *Consumer { + ctx, cancel := context.WithCancel(context.Background()) + return &Consumer{ cfg: cfg, serializer: ser, conn: conn, + readers: make([]*kafka.Reader, 0), + ctx: ctx, + cancel: cancel, + logger: logger, } } func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - readerConfig := kafka.ReaderConfig{ - GroupID: c.cfg.Options.Group, - Topic: c.cfg.Options.Topic, - StartOffset: c.startOffset(c.cfg.Options.Offset), - CommitInterval: c.cfg.Options.CommitInterval, - MinBytes: minBytes, - MaxBytes: maxBytes, - ReadLagInterval: readLagInterval, + for _, topic := range c.cfg.Options.Topics { + readerConfig := kafka.ReaderConfig{ + GroupID: c.cfg.Options.Group, + Topic: topic, + CommitInterval: c.cfg.Options.Commit.Interval, + MinBytes: minBytes, + MaxBytes: maxBytes, + ReadLagInterval: readLagInterval, + SessionTimeout: sessionTimeout, + RebalanceTimeout: rebalanceTimeout, + HeartbeatInterval: heartbeatInterval, + MaxWait: time.Second, + } - SessionTimeout: sessionTimeout, - RebalanceTimeout: rebalanceTimeout, - HeartbeatInterval: heartbeatInterval, - MaxWait: time.Second, - } + switch c.cfg.Options.Rebalance.Strategy { + case "range": + readerConfig.GroupBalancers = []kafka.GroupBalancer{ + kafka.RangeGroupBalancer{}, + } + case "roundrobin": + readerConfig.GroupBalancers = []kafka.GroupBalancer{ + kafka.RoundRobinGroupBalancer{}, + } + } - r := c.conn.CreateReader(readerConfig) - defer r.Close() + c.configureOffset(&readerConfig) + + reader := c.conn.CreateReader(readerConfig) + c.readers = append(c.readers, reader) + } jobs := make(chan job) c.startWorkerPool(ctx, jobs, handler) - go c.fetchMessages(ctx, r, jobs) + for _, reader := range c.readers { + c.wg.Add(1) + go func(r *kafka.Reader) { + defer c.wg.Done() + c.fetchMessages(ctx, r, jobs) + }(reader) + } + + if c.cfg.Options.Commit.Strategy == "batch" || c.cfg.Options.Commit.Strategy == "deferred" { + go c.startBatchCommitter(ctx) + } <-ctx.Done() + for _, reader := range c.readers { + if err := reader.Close(); err != nil { + c.logger.ErrorContext(ctx, "Failed to close Kafka reader", "error", err) + } + } + + c.cancel() + c.wg.Wait() + return ctx.Err() } +func (c *Consumer) configureOffset(config *kafka.ReaderConfig) { + switch c.cfg.Options.OffsetConfig.Type { + case "earliest": + config.StartOffset = kafka.FirstOffset + case "specific": + config.StartOffset = c.cfg.Options.OffsetConfig.Value + default: + config.StartOffset = kafka.LastOffset + } +} + func (c *Consumer) startWorkerPool( ctx context.Context, jobs chan job, handler func(context.Context, api.Envelope) error, ) { - poolSize := c.cfg.Options.ConsumerPoolSize + poolSize := c.cfg.Options.Pool.Size if poolSize <= 0 { poolSize = defaultPoolSize } - for i := range make([]struct{}, poolSize) { - go func(_ int) { - for j := range jobs { - c.handleMessage(ctx, j.r, j.msg, handler) + for i := 0; i < poolSize; i++ { + c.wg.Add(1) + go func() { + defer c.wg.Done() + + for { + select { + case <-ctx.Done(): + return + case j, ok := <-jobs: + if !ok { + return + } + c.handleMessage(ctx, j.r, j.msg, handler) + } } - }(i) + }() + } + + if c.cfg.Options.Pool.Dynamic { + go c.manageWorkerPool(ctx, jobs, handler) } } -func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan job) { +func (c *Consumer) manageWorkerPool( + ctx context.Context, + jobs chan job, + handler func(context.Context, api.Envelope) error, +) { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + currentSize := c.cfg.Options.Pool.Size + minSize := c.cfg.Options.Pool.MinSize + maxSize := c.cfg.Options.Pool.MaxSize + for { select { case <-ctx.Done(): - close(jobs) + return + case <-ticker.C: + if len(jobs) > currentSize && currentSize < maxSize { + toAdd := min(maxSize-currentSize, 5) + for i := 0; i < toAdd; i++ { + c.wg.Add(1) + go func() { + defer c.wg.Done() + + for { + select { + case <-ctx.Done(): + return + case j, ok := <-jobs: + if !ok { + return + } + c.handleMessage(ctx, j.r, j.msg, handler) + } + } + }() + } + currentSize += toAdd + c.logger.DebugContext(ctx, "Increased worker pool size", "new_size", currentSize) + } else if len(jobs) == 0 && currentSize > minSize { + currentSize = max(currentSize-5, minSize) + c.logger.DebugContext(ctx, "Decreased worker pool size", "new_size", currentSize) + } + } + } +} +func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan job) { + for { + select { + case <-ctx.Done(): return default: msg, err := r.FetchMessage(ctx) @@ -96,10 +221,17 @@ func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan return } + c.logger.ErrorContext(ctx, "Failed to fetch message", "error", err) + time.Sleep(100 * time.Millisecond) + continue } - jobs <- job{r: r, msg: msg} + select { + case <-ctx.Done(): + return + case jobs <- job{r: r, msg: msg}: + } } } } @@ -112,7 +244,9 @@ func (c *Consumer) handleMessage( ) { env, err := c.serializer.Unmarshal(msg.Value, c.headerMap(msg.Headers)) if err != nil { - _ = r.CommitMessages(ctx, msg) + c.logger.ErrorContext(ctx, "Failed to unmarshal message", "error", err) + + c.commitMessage(ctx, r, msg) return } @@ -120,25 +254,109 @@ func (c *Consumer) handleMessage( env = env.WithStamp(stamps.ReceivedStamp{Transport: c.cfg.Name}) if handlerErr := handler(ctx, env); handlerErr != nil { - _ = r.CommitMessages(ctx, msg) + c.logger.ErrorContext(ctx, "Handler failed", "error", handlerErr) + c.commitMessage(ctx, r, msg) return } - _ = r.CommitMessages(ctx, msg) + c.commitMessage(ctx, r, msg) } -type job struct { - r *kafka.Reader - msg kafka.Message +func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka.Message) { + switch c.cfg.Options.Commit.Strategy { + case "auto": + if err := r.CommitMessages(ctx, msg); err != nil { + c.logger.ErrorContext(ctx, "Failed to commit message", + "topic", msg.Topic, + "partition", msg.Partition, + "offset", msg.Offset, + "error", err) + } + case "manual": + case "batch": + c.batchMutex.Lock() + c.batchMessages = append(c.batchMessages, msg) + + msgKey := fmt.Sprintf("%s-%d-%d", msg.Topic, msg.Partition, msg.Offset) + c.deferredCommits.Store(msgKey, messageWithReader{message: msg, reader: r}) + + if len(c.batchMessages) >= c.cfg.Options.Commit.BatchSize { + c.batchMutex.Unlock() + c.commitBatch() + } else { + c.batchMutex.Unlock() + } + case "deferred": + msgKey := fmt.Sprintf("%s-%d-%d", msg.Topic, msg.Partition, msg.Offset) + c.deferredCommits.Store(msgKey, messageWithReader{message: msg, reader: r}) + } +} + +func (c *Consumer) startBatchCommitter(ctx context.Context) { + ticker := time.NewTicker(c.cfg.Options.Commit.Interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.commitBatch() + } + } } -func (c *Consumer) startOffset(offset string) int64 { - if offset == "earliest" { - return kafka.FirstOffset +func (c *Consumer) commitBatch() { + c.batchMutex.Lock() + defer c.batchMutex.Unlock() + + readerMessages := make(map[*kafka.Reader][]kafka.Message) + + c.deferredCommits.Range(func(_, value any) bool { + if msgWithReader, ok := value.(messageWithReader); ok { + readerMessages[msgWithReader.reader] = append(readerMessages[msgWithReader.reader], msgWithReader.message) + } + + return true + }) + + for reader, messages := range readerMessages { + if len(messages) == 0 { + continue + } + + partitionOffsets := make(map[int]kafka.Message) + for _, msg := range messages { + currentMax, exists := partitionOffsets[msg.Partition] + if !exists || currentMax.Offset < msg.Offset { + partitionOffsets[msg.Partition] = msg + } + } + + for _, msg := range partitionOffsets { + if err := reader.CommitMessages(context.Background(), msg); err != nil { + c.logger.Error("Failed to commit message batch", + "topic", msg.Topic, + "partition", msg.Partition, + "offset", msg.Offset, + "error", err.Error()) + } else { + for _, commitedMsg := range messages { + if commitedMsg.Partition == msg.Partition && commitedMsg.Offset <= msg.Offset { + c.deferredCommits.Delete(fmt.Sprintf("%s-%d-%d", commitedMsg.Topic, commitedMsg.Partition, commitedMsg.Offset)) + } + } + } + } } - return kafka.LastOffset + c.batchMessages = nil +} + +type job struct { + r *kafka.Reader + msg kafka.Message } func (c *Consumer) headerMap(headers []kafka.Header) map[string]string { diff --git a/transport/kafka/factory_test.go b/transport/kafka/factory_test.go index 147d0aa..c6f43e3 100644 --- a/transport/kafka/factory_test.go +++ b/transport/kafka/factory_test.go @@ -85,11 +85,8 @@ func TestTransportFactory_Create(t *testing.T) { name := "test-kafka" dsn := "kafka://non-existent-host:9092" options := kafka.OptionsConfig{ - ConsumerPoolSize: 5, - Offset: "earliest", - Group: "test-group", - Topic: "test-topic", - CommitInterval: 1000000000, + Topics: []string{"test-topic"}, + Group: "test-group", } optionsBytes, err := yaml.Marshal(options) diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index b57a76f..7793d68 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -2,25 +2,30 @@ package kafka import ( "context" + "errors" "fmt" + "log/slog" "time" "github.com/segmentio/kafka-go" "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" ) type Producer struct { - writer *kafka.Writer cfg TransportConfig serializer api.Serializer + conn *Connection + logger *slog.Logger } -func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection) (*Producer, error) { +func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) (*Producer, error) { return &Producer{ cfg: cfg, serializer: ser, - writer: conn.CreateWriter(cfg.Options.Topic), + conn: conn, + logger: logger, }, nil } @@ -41,9 +46,54 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { Time: time.Now(), } - if writeErr := p.writer.WriteMessages(ctx, msg); writeErr != nil { - return fmt.Errorf("producer failed to write messages: %w", writeErr) + key, keyErr := p.extractMessageKey(env) + if keyErr == nil && len(key) > 0 { + msg.Key = key + } + + topics := p.cfg.Options.Topics + + if len(topics) == 0 { + return errors.New("no topics configured for kafka transport") + } + + for _, topic := range topics { + writer := p.conn.CreateWriter(topic) + + if p.cfg.Options.Key.Strategy != "none" { + writer.Balancer = &kafka.Hash{} + } + + if writeErr := writer.WriteMessages(ctx, msg); writeErr != nil { + return fmt.Errorf("producer failed to write messages: %w", writeErr) + } + + p.logger.DebugContext(ctx, "message sent to kafka topic", + slog.String("topic", topic), + slog.String("message_type", fmt.Sprintf("%T", env.Message()))) + + err := writer.Close() + if err != nil { + return err + } } return nil } + +func (p *Producer) extractMessageKey(env api.Envelope) ([]byte, error) { + switch p.cfg.Options.Key.Strategy { + case "none": + return nil, nil + case "message_id": + for _, s := range env.Stamps() { + if msgIDStamp, ok := s.(stamps.MessageIDStamp); ok { + return []byte(msgIDStamp.MessageID), nil + } + } + + return nil, fmt.Errorf("message_id stamp not found") + default: + return nil, fmt.Errorf("unknown key strategy: %s", p.cfg.Options.Key.Strategy) + } +} diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index ebd0ce4..044f289 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -37,12 +37,12 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L ser := serializer.NewSerializer(resolver) - producer, err := NewProducer(cfg, ser, conn) + producer, err := NewProducer(cfg, ser, conn, logger) if err != nil { return nil, fmt.Errorf("failed to create kafka producer: %w", err) } - consumer := NewConsumer(cfg, ser, conn) + consumer := NewConsumer(cfg, ser, conn, logger) return &Transport{ cfg: cfg, From 402691c2d6468967b6bcf79fe6a9629922abebe1 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Mon, 4 Aug 2025 13:36:19 +0700 Subject: [PATCH 08/17] refactor: code style --- transport/kafka/consumer.go | 145 +++++++++++++++++++++--------------- transport/kafka/producer.go | 8 +- 2 files changed, 87 insertions(+), 66 deletions(-) diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index dcd4e0d..b8bafd3 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -21,6 +21,10 @@ const ( heartbeatInterval = 2 * time.Second defaultPoolSize = 10 readLagInterval = -1 + + workerPoolCheckInterval = 30 * time.Second + workerBatchSize = 5 + errorBackoffDelay = 100 * time.Millisecond ) type Consumer struct { @@ -138,23 +142,9 @@ func (c *Consumer) startWorkerPool( poolSize = defaultPoolSize } - for i := 0; i < poolSize; i++ { + for range poolSize { c.wg.Add(1) - go func() { - defer c.wg.Done() - - for { - select { - case <-ctx.Done(): - return - case j, ok := <-jobs: - if !ok { - return - } - c.handleMessage(ctx, j.r, j.msg, handler) - } - } - }() + go c.startWorker(ctx, jobs, handler) } if c.cfg.Options.Pool.Dynamic { @@ -167,7 +157,7 @@ func (c *Consumer) manageWorkerPool( jobs chan job, handler func(context.Context, api.Envelope) error, ) { - ticker := time.NewTicker(30 * time.Second) + ticker := time.NewTicker(workerPoolCheckInterval) defer ticker.Stop() currentSize := c.cfg.Options.Pool.Size @@ -180,35 +170,39 @@ func (c *Consumer) manageWorkerPool( return case <-ticker.C: if len(jobs) > currentSize && currentSize < maxSize { - toAdd := min(maxSize-currentSize, 5) - for i := 0; i < toAdd; i++ { + toAdd := min(maxSize-currentSize, workerBatchSize) + + for range toAdd { c.wg.Add(1) - go func() { - defer c.wg.Done() - - for { - select { - case <-ctx.Done(): - return - case j, ok := <-jobs: - if !ok { - return - } - c.handleMessage(ctx, j.r, j.msg, handler) - } - } - }() + go c.startWorker(ctx, jobs, handler) } + currentSize += toAdd c.logger.DebugContext(ctx, "Increased worker pool size", "new_size", currentSize) } else if len(jobs) == 0 && currentSize > minSize { - currentSize = max(currentSize-5, minSize) + currentSize = max(currentSize-workerBatchSize, minSize) c.logger.DebugContext(ctx, "Decreased worker pool size", "new_size", currentSize) } } } } +func (c *Consumer) startWorker(ctx context.Context, jobs chan job, handler func(context.Context, api.Envelope) error) { + defer c.wg.Done() + + for { + select { + case <-ctx.Done(): + return + case j, ok := <-jobs: + if !ok { + return + } + c.handleMessage(ctx, j.r, j.msg, handler) + } + } +} + func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan job) { for { select { @@ -222,7 +216,7 @@ func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan } c.logger.ErrorContext(ctx, "Failed to fetch message", "error", err) - time.Sleep(100 * time.Millisecond) + time.Sleep(errorBackoffDelay) continue } @@ -311,6 +305,22 @@ func (c *Consumer) commitBatch() { c.batchMutex.Lock() defer c.batchMutex.Unlock() + readerMessages := c.groupMessagesByReader() + + for reader, messages := range readerMessages { + if len(messages) == 0 { + continue + } + + partitionOffsets := c.findMaxOffsetPerPartition(messages) + + c.commitMessagesAndCleanup(reader, messages, partitionOffsets) + } + + c.batchMessages = c.batchMessages[:0] +} + +func (c *Consumer) groupMessagesByReader() map[*kafka.Reader][]kafka.Message { readerMessages := make(map[*kafka.Reader][]kafka.Message) c.deferredCommits.Range(func(_, value any) bool { @@ -321,37 +331,48 @@ func (c *Consumer) commitBatch() { return true }) - for reader, messages := range readerMessages { - if len(messages) == 0 { - continue - } + return readerMessages +} - partitionOffsets := make(map[int]kafka.Message) - for _, msg := range messages { - currentMax, exists := partitionOffsets[msg.Partition] - if !exists || currentMax.Offset < msg.Offset { - partitionOffsets[msg.Partition] = msg - } +func (c *Consumer) findMaxOffsetPerPartition(messages []kafka.Message) map[int]kafka.Message { + partitionOffsets := make(map[int]kafka.Message) + + for _, msg := range messages { + currentMax, exists := partitionOffsets[msg.Partition] + if !exists || currentMax.Offset < msg.Offset { + partitionOffsets[msg.Partition] = msg } + } - for _, msg := range partitionOffsets { - if err := reader.CommitMessages(context.Background(), msg); err != nil { - c.logger.Error("Failed to commit message batch", - "topic", msg.Topic, - "partition", msg.Partition, - "offset", msg.Offset, - "error", err.Error()) - } else { - for _, commitedMsg := range messages { - if commitedMsg.Partition == msg.Partition && commitedMsg.Offset <= msg.Offset { - c.deferredCommits.Delete(fmt.Sprintf("%s-%d-%d", commitedMsg.Topic, commitedMsg.Partition, commitedMsg.Offset)) - } - } - } + return partitionOffsets +} + +func (c *Consumer) commitMessagesAndCleanup( + reader *kafka.Reader, + messages []kafka.Message, + partitionOffsets map[int]kafka.Message, +) { + for _, msg := range partitionOffsets { + if err := reader.CommitMessages(context.Background(), msg); err != nil { + c.logger.Error("Failed to commit message batch", + "topic", msg.Topic, + "partition", msg.Partition, + "offset", msg.Offset, + "error", err.Error()) + } else { + c.cleanupCommittedMessages(messages, msg) } } +} - c.batchMessages = nil +func (c *Consumer) cleanupCommittedMessages(messages []kafka.Message, committedMsg kafka.Message) { + for _, commitedMsg := range messages { + if commitedMsg.Partition == committedMsg.Partition && commitedMsg.Offset <= committedMsg.Offset { + c.deferredCommits.Delete( + fmt.Sprintf("%s-%d-%d", commitedMsg.Topic, commitedMsg.Partition, commitedMsg.Offset), + ) + } + } } type job struct { @@ -360,7 +381,7 @@ type job struct { } func (c *Consumer) headerMap(headers []kafka.Header) map[string]string { - m := make(map[string]string, len(headers)) + m := make(map[string]string) for _, h := range headers { m[h.Key] = string(h.Value) } diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index 7793d68..3079aca 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -72,9 +72,9 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { slog.String("topic", topic), slog.String("message_type", fmt.Sprintf("%T", env.Message()))) - err := writer.Close() - if err != nil { - return err + closeErr := writer.Close() + if closeErr != nil { + return closeErr } } @@ -92,7 +92,7 @@ func (p *Producer) extractMessageKey(env api.Envelope) ([]byte, error) { } } - return nil, fmt.Errorf("message_id stamp not found") + return nil, errors.New("message_id stamp not found") default: return nil, fmt.Errorf("unknown key strategy: %s", p.cfg.Options.Key.Strategy) } From f1fbec45f9e5383c570168ba5fb621433d73ba3b Mon Sep 17 00:00:00 2001 From: Gerfey Date: Mon, 4 Aug 2025 13:38:59 +0700 Subject: [PATCH 09/17] =?UTF-8?q?docs:=20=D1=83=D0=B4=D0=B0=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D1=8D=D0=BC=D0=BE=D0=B4=D0=B7=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 30 +++++++++++++++--------------- README.ru.md | 30 +++++++++++++++--------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index b07e4c2..c8631cc 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,13 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/Gerfey/messenger.svg)](https://pkg.go.dev/github.com/Gerfey/messenger) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -> ⚠️ `v0.7.0` is a pre-release version — feel free to test and report issues! +> `v0.7.0` is a pre-release version — feel free to test and report issues! -> 📚 Full documentation available in the [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) +> Full documentation available in the [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) 🇷🇺 [Русская версия](README.ru.md) -## ✨ Features +## Features - **Multiple Transports**: AMQP (RabbitMQ), In-Memory (sync) - **Middleware Chain**: Extensible middleware system for message processing - **Event-Driven**: Built-in event dispatcher for lifecycle hooks @@ -22,13 +22,13 @@ - **Stamps System**: Metadata attachment for message tracking - **YAML Configuration**: Easy configuration management with `%env(...)%` support -## 📦 Installation +## Installation > Requires Go 1.24+ ```bash go get github.com/gerfey/messenger@v0.7.0 ``` -## 🚀 Quick Start +## Quick Start ### Define Your Message @@ -83,17 +83,17 @@ bus, _ := m.GetDefaultBus() _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) ``` -## 🔍 More Examples +## More Examples -* ✅ Commands with void return -* ✅ Queries with return value access -* ✅ Retry and Dead Letter Queue -* ✅ Custom Middleware and Transports -* ✅ Event Listeners and Lifecycle Hooks +* Commands with void return +* Queries with return value access +* Retry and Dead Letter Queue +* Custom Middleware and Transports +* Event Listeners and Lifecycle Hooks > See [Usage Scenarios](https://github.com/Gerfey/messenger/wiki/Usage-Scenarios) for commands, queries, return values and advanced use-cases. -## 🤝 Contributing +## Contributing 1. Fork the repository 2. Create your feature branch (`git checkout -b feature/amazing-feature`) @@ -101,15 +101,15 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) 4. Push to the branch (`git push origin feature/amazing-feature`) 5. Open a Pull Request -## ⚖️ License +## ⚖License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. -## ⭐️ Support +## Support If you find this project useful, please consider starring ⭐️ it and sharing with others! -## 🙏 Acknowledgments +## Acknowledgments - Inspired by [Symfony Messenger](https://symfony.com/doc/current/messenger.html) - Built with ❤️ for the Go community diff --git a/README.ru.md b/README.ru.md index c4c6bf7..7718829 100644 --- a/README.ru.md +++ b/README.ru.md @@ -7,15 +7,15 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/Gerfey/messenger.svg)](https://pkg.go.dev/github.com/Gerfey/messenger) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -> ⚠️ Версия `v0.7.0` — это пре-релиз. Тестируйте и сообщайте о багах! +> Версия `v0.7.0` — это пре-релиз. Тестируйте и сообщайте о багах! -> 📚 Полная документация доступна на [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) +> Полная документация доступна на [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) 🇬🇧 [English README](README.md) --- -## ✨ Возможности +## Возможности - **Множественные транспорты**: AMQP (RabbitMQ), In-Memory (`sync`) - **Цепочка middleware**: Расширяемая система промежуточной обработки - **Событийный движок**: Встроенный dispatcher событий жизненного цикла @@ -24,13 +24,13 @@ - **Система метаданных (Stamps)**: Для трассировки и поведения сообщений - **YAML-конфигурация**: С поддержкой переменных окружения `%env(...)%` -## 📦 Установка +## Установка > Требуется Go версии **1.24+** ```bash go get github.com/gerfey/messenger@v0.7.0 ``` -## 🚀 Быстрый старт +## Быстрый старт ### Определите сообщение @@ -85,17 +85,17 @@ bus, _ := m.GetDefaultBus() _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) ``` -## 🔍 Больше примеров +## Больше примеров -* ✅ Команды без возврата значения -* ✅ Запросы с возвратом результата -* ✅ Повторные попытки и Dead Letter Queue -* ✅ Пользовательские middleware и транспорты -* ✅ Слушатели событий и хуки жизненного цикла +* Команды без возврата значения +* Запросы с возвратом результата +* Повторные попытки и Dead Letter Queue +* Пользовательские middleware и транспорты +* Слушатели событий и хуки жизненного цикла > Смотри [Сценарии использования](https://github.com/Gerfey/messenger/wiki/Сценарии-использования). -## 🤝 Как внести вклад +## Как внести вклад 1. Форкните репозиторий 2. Создайте новую ветку (`git checkout -b feature/amazing-feature`) @@ -103,15 +103,15 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) 4. Запушьте изменения (`git push origin feature/amazing-feature`) 5. Откройте Pull Request -## ⚖️ Лицензия +## Лицензия Проект лицензирован под [MIT](LICENSE). -## ⭐️ Поддержка +## Поддержка Если вам полезен этот проект — поставьте ⭐️ и расскажите другим! -## 🙏 Благодарности +## Благодарности - Вдохновлено [Symfony Messenger](https://symfony.com/doc/current/messenger.html) - Сделано с ❤️ для сообщества Go From 826e02f86cbef5074d743424a5264a13aae042d4 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Mon, 4 Aug 2025 13:52:15 +0700 Subject: [PATCH 10/17] feat: added a dynamic pool of workers for the AMQP consumer --- examples/messenger/messenger.yaml | 6 ++- transport/amqp/config.go | 20 ++++---- transport/amqp/consumer.go | 84 ++++++++++++++++++++++++++++--- 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/examples/messenger/messenger.yaml b/examples/messenger/messenger.yaml index 130078e..72d1501 100644 --- a/examples/messenger/messenger.yaml +++ b/examples/messenger/messenger.yaml @@ -15,7 +15,11 @@ transports: max_delay: 5s options: auto_setup: true - consumer_pool_size: 10 + pool: + size: 10 + min_size: 5 + max_size: 20 + dynamic: true exchange: name: test.exchange type: topic diff --git a/transport/amqp/config.go b/transport/amqp/config.go index 391aa60..4d705e5 100644 --- a/transport/amqp/config.go +++ b/transport/amqp/config.go @@ -1,9 +1,5 @@ package amqp -import ( - "time" -) - type TransportConfig struct { Name string DSN string @@ -11,11 +7,17 @@ type TransportConfig struct { } type OptionsConfig struct { - AutoSetup bool `yaml:"auto_setup" default:"false"` - ConsumerPoolSize int `yaml:"consumer_pool_size" default:"10"` - CommitInterval time.Duration `yaml:"commit_interval" default:"10"` - Exchange ExchangeConfig `yaml:"exchange"` - Queues map[string]Queue `yaml:"queues"` + AutoSetup bool `yaml:"auto_setup" default:"false"` + Pool PoolConfig `yaml:"pool"` + Exchange ExchangeConfig `yaml:"exchange"` + Queues map[string]Queue `yaml:"queues"` +} + +type PoolConfig struct { + Size int `yaml:"size" default:"10"` + MinSize int `yaml:"min_size" default:"5"` + MaxSize int `yaml:"max_size" default:"20"` + Dynamic bool `yaml:"dynamic" default:"false"` } type ExchangeConfig struct { diff --git a/transport/amqp/consumer.go b/transport/amqp/consumer.go index a4aae18..6e1ea2b 100644 --- a/transport/amqp/consumer.go +++ b/transport/amqp/consumer.go @@ -3,6 +3,9 @@ package amqp import ( "context" "fmt" + "log/slog" + "sync" + "time" amqp "github.com/rabbitmq/amqp091-go" @@ -10,10 +13,18 @@ import ( "github.com/gerfey/messenger/core/stamps" ) +const ( + defaultPoolSize = 10 + workerPoolCheckInterval = 30 * time.Second + workerBatchSize = 5 +) + type Consumer struct { conn *Connection cfg TransportConfig serializer api.Serializer + wg sync.WaitGroup + logger *slog.Logger } func NewConsumer(conn *Connection, cfg TransportConfig, serializer api.Serializer) *Consumer { @@ -21,6 +32,7 @@ func NewConsumer(conn *Connection, cfg TransportConfig, serializer api.Serialize conn: conn, cfg: cfg, serializer: serializer, + logger: slog.Default(), } } @@ -42,6 +54,8 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap } <-ctx.Done() + close(jobs) + c.wg.Wait() return ctx.Err() } @@ -51,17 +65,73 @@ func (c *Consumer) startWorkerPool( jobs chan job, handler func(context.Context, api.Envelope) error, ) { - poolSize := c.cfg.Options.ConsumerPoolSize + poolSize := c.cfg.Options.Pool.Size if poolSize <= 0 { - poolSize = 10 + poolSize = defaultPoolSize } for range poolSize { - go func() { - for j := range jobs { - c.handleDelivery(ctx, j.d, handler) + c.wg.Add(1) + go c.startWorker(ctx, jobs, handler) + } + + if c.cfg.Options.Pool.Dynamic { + go c.manageWorkerPool(ctx, jobs, handler) + } +} + +func (c *Consumer) manageWorkerPool( + ctx context.Context, + jobs chan job, + handler func(context.Context, api.Envelope) error, +) { + ticker := time.NewTicker(workerPoolCheckInterval) + defer ticker.Stop() + + currentSize := c.cfg.Options.Pool.Size + minSize := c.cfg.Options.Pool.MinSize + maxSize := c.cfg.Options.Pool.MaxSize + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if len(jobs) > currentSize && currentSize < maxSize { + toAdd := min(maxSize-currentSize, workerBatchSize) + + for range toAdd { + c.wg.Add(1) + go c.startWorker(ctx, jobs, handler) + } + + currentSize += toAdd + c.logger.DebugContext(ctx, "Increased worker pool size", "new_size", currentSize) + } else if len(jobs) == 0 && currentSize > minSize { + currentSize = max(currentSize-workerBatchSize, minSize) + c.logger.DebugContext(ctx, "Decreased worker pool size", "new_size", currentSize) } - }() + } + } +} + +func (c *Consumer) startWorker( + ctx context.Context, + jobs chan job, + handler func(context.Context, api.Envelope) error, +) { + defer c.wg.Done() + + for { + select { + case <-ctx.Done(): + return + case j, ok := <-jobs: + if !ok { + return + } + c.handleDelivery(ctx, j.d, handler) + } } } @@ -91,8 +161,6 @@ func (c *Consumer) processQueueMessages(ctx context.Context, jobs chan job, mess for { select { case <-ctx.Done(): - close(jobs) - return case d, ok := <-messages: if !ok { From 2671c6726c61854ad67d265d719f9cf632f03956 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Mon, 4 Aug 2025 21:09:02 +0700 Subject: [PATCH 11/17] feat: added transport Redis --- api/transport.go | 4 + builder/builder.go | 2 + .../redis_transport/handler/hello_handler.go | 20 +++ examples/redis_transport/message/hello.go | 9 ++ examples/redis_transport/messenger.yaml | 32 ++++ examples/redis_transport/redis_transport.go | 66 +++++++++ go.mod | 3 + go.sum | 6 + transport/amqp/factory.go | 4 +- transport/amqp/transport.go | 39 ++--- transport/kafka/factory.go | 4 +- transport/kafka/factory_test.go | 1 - transport/kafka/transport.go | 4 +- transport/manager.go | 8 + transport/redis/config.go | 14 ++ transport/redis/connection.go | 48 ++++++ transport/redis/consumer.go | 139 ++++++++++++++++++ transport/redis/factory.go | 47 ++++++ transport/redis/producer.go | 77 ++++++++++ transport/redis/transport.go | 75 ++++++++++ 20 files changed, 576 insertions(+), 26 deletions(-) create mode 100644 examples/redis_transport/handler/hello_handler.go create mode 100644 examples/redis_transport/message/hello.go create mode 100644 examples/redis_transport/messenger.yaml create mode 100644 examples/redis_transport/redis_transport.go create mode 100644 transport/redis/config.go create mode 100644 transport/redis/connection.go create mode 100644 transport/redis/consumer.go create mode 100644 transport/redis/factory.go create mode 100644 transport/redis/producer.go create mode 100644 transport/redis/transport.go diff --git a/api/transport.go b/api/transport.go index d99806a..b23d9bb 100644 --- a/api/transport.go +++ b/api/transport.go @@ -39,3 +39,7 @@ type TransportFactory interface { type RoutedMessage interface { RoutingKey() string } + +type Setupable interface { + Setup(ctx context.Context) error +} diff --git a/builder/builder.go b/builder/builder.go index 2a9a89a..ed4fc79 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -22,6 +22,7 @@ import ( "github.com/gerfey/messenger/transport/amqp" "github.com/gerfey/messenger/transport/inmemory" "github.com/gerfey/messenger/transport/kafka" + "github.com/gerfey/messenger/transport/redis" "github.com/gerfey/messenger/transport/sync" ) @@ -47,6 +48,7 @@ func NewBuilder(cfg *config.MessengerConfig, logger *slog.Logger) api.Builder { inmemory.NewTransportFactory(logger, resolver), sync.NewTransportFactory(logger, busLocator), kafka.NewTransportFactory(logger, resolver), + redis.NewTransportFactory(logger, resolver), ) return &Builder{ diff --git a/examples/redis_transport/handler/hello_handler.go b/examples/redis_transport/handler/hello_handler.go new file mode 100644 index 0000000..0c7cd7d --- /dev/null +++ b/examples/redis_transport/handler/hello_handler.go @@ -0,0 +1,20 @@ +package handler + +import ( + "context" + "fmt" + + "github.com/gerfey/messenger/examples/redis_transport/message" +) + +type ExampleHelloHandler struct{} + +func (u *ExampleHelloHandler) Handle(_ context.Context, msg *message.ExampleHelloMessage) error { + fmt.Printf("Handled: Text=%v\n", msg.Text) + + return nil +} + +func (u *ExampleHelloHandler) GetBusName() string { + return "default" +} diff --git a/examples/redis_transport/message/hello.go b/examples/redis_transport/message/hello.go new file mode 100644 index 0000000..a7877ee --- /dev/null +++ b/examples/redis_transport/message/hello.go @@ -0,0 +1,9 @@ +package message + +type ExampleHelloMessage struct { + Text string +} + +func (m *ExampleHelloMessage) RoutingKey() string { + return "test_routing_key" +} diff --git a/examples/redis_transport/messenger.yaml b/examples/redis_transport/messenger.yaml new file mode 100644 index 0000000..a9b2867 --- /dev/null +++ b/examples/redis_transport/messenger.yaml @@ -0,0 +1,32 @@ +default_bus: default +failure_transport: failed_messages + +buses: + default: ~ + +transports: + redis: + dsn: "redis://localhost:6379/0" + retry_strategy: + max_retries: 5 + delay: 500ms + multiplier: 2 + max_delay: 5s + options: + auto_setup: true + stream: stream-messages + group: worker-group + consumer: worker-1 + + failed_messages: + dsn: "amqp://guest:guest@localhost:5672/" + options: + auto_setup: true + exchange: + name: failed_exchange + type: fanout + queues: + failed_messages_queue: ~ + +routing: + message.ExampleHelloMessage: redis diff --git a/examples/redis_transport/redis_transport.go b/examples/redis_transport/redis_transport.go new file mode 100644 index 0000000..ed9c097 --- /dev/null +++ b/examples/redis_transport/redis_transport.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/examples/redis_transport/handler" + "github.com/gerfey/messenger/examples/redis_transport/message" +) + +const ( + waitDurationSeconds = 20 +) + +func main() { + ctx := context.Background() + + log := slog.Default() + + cfg, err := config.LoadConfig("./examples/redis_transport/messenger.yaml") + if err != nil { + log.Error("ERROR load config", "error", err) + + return + } + + b := builder.NewBuilder(cfg, log) + + _ = b.RegisterHandler(&handler.ExampleHelloHandler{}) + + messenger, err := b.Build() + if err != nil { + log.Error("failed to build messenger", "error", err) + + return + } + + go func() { + if runErr := messenger.Run(ctx); runErr != nil { + log.Error("messenger run failed", "error", runErr) + + return + } + }() + + messengerBus, err := messenger.GetDefaultBus() + if err != nil { + log.Error("failed to get default bus", "error", err) + + return + } + + _, err = messengerBus.Dispatch(ctx, &message.ExampleHelloMessage{ + Text: "Hello World", + }) + if err != nil { + log.Error("failed to dispatch message", "error", err) + + return + } + + time.Sleep(waitDurationSeconds * time.Second) +} diff --git a/go.mod b/go.mod index c47cb2a..4d35b79 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,14 @@ require ( require ( github.com/BurntSushi/toml v1.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/google/uuid v1.6.0 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/redis/go-redis/v9 v9.11.0 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index 5b63c18..5008938 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,14 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= @@ -19,6 +23,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs= +github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/transport/amqp/factory.go b/transport/amqp/factory.go index 65dbd6d..ba98c49 100644 --- a/transport/amqp/factory.go +++ b/transport/amqp/factory.go @@ -30,11 +30,11 @@ func (f *TransportFactory) Supports(dsn string) bool { func (f *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { var opts OptionsConfig if err := defaults.Set(&opts); err != nil { - return nil, fmt.Errorf("amqp: set defaults: %w", err) + return nil, fmt.Errorf("set defaults: %w", err) } if err := yaml.Unmarshal(options, &opts); err != nil { - return nil, fmt.Errorf("amqp: unmarshal options: %w", err) + return nil, fmt.Errorf("unmarshal options: %w", err) } cfg := TransportConfig{ diff --git a/transport/amqp/transport.go b/transport/amqp/transport.go index f85a34d..fd93fab 100644 --- a/transport/amqp/transport.go +++ b/transport/amqp/transport.go @@ -34,7 +34,7 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L cons := NewConsumer(conn, cfg, ser) ret := NewRetry(conn, cfg, ser) - transport := &Transport{ + return &Transport{ cfg: cfg, publisher: pub, consumer: cons, @@ -42,19 +42,7 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L serializer: ser, conn: conn, logger: logger, - } - - if cfg.Options.AutoSetup { - if setupErr := transport.setup(); setupErr != nil { - logger.Error("failed to setup AMQP transport", "transport", cfg.Name, "error", setupErr) - - return nil, setupErr - } - - logger.Debug("AMQP transport setup completed", "transport", cfg.Name) - } - - return transport, nil + }, nil } func (t *Transport) Name() string { @@ -73,10 +61,14 @@ func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { return t.retry.Retry(ctx, env) } -func (t *Transport) setup() error { +func (t *Transport) Setup(ctx context.Context) error { + if !t.cfg.Options.AutoSetup { + return nil + } + ch, err := t.conn.Channel() if err != nil { - t.logger.Error("failed to open channel", "error", err) + t.logger.ErrorContext(ctx, "failed to open channel", "error", err) return fmt.Errorf("failed to open channel: %w", err) } @@ -94,7 +86,7 @@ func (t *Transport) setup() error { nil, ) if err != nil { - t.logger.Error("failed to declare exchange", "exchange", t.cfg.Options.Exchange.Name, "error", err) + t.logger.ErrorContext(ctx, "failed to declare exchange", "exchange", t.cfg.Options.Exchange.Name, "error", err) return fmt.Errorf("failed to declare exchange: %w", err) } @@ -109,7 +101,7 @@ func (t *Transport) setup() error { nil, ) if err != nil { - t.logger.Error("declare queue", "queue", queueName, "error", err) + t.logger.ErrorContext(ctx, "declare queue", "queue", queueName, "error", err) return fmt.Errorf("declare queue: %w", err) } @@ -129,7 +121,16 @@ func (t *Transport) setup() error { nil, ) if bindErr != nil { - t.logger.Error("bind queue", "queue", queueName, "binding_key", bindingKey, "error", bindErr) + t.logger.ErrorContext( + ctx, + "bind queue", + "queue", + queueName, + "binding_key", + bindingKey, + "error", + bindErr, + ) return fmt.Errorf("bind queue: %w", bindErr) } diff --git a/transport/kafka/factory.go b/transport/kafka/factory.go index a2bc117..682db81 100644 --- a/transport/kafka/factory.go +++ b/transport/kafka/factory.go @@ -30,11 +30,11 @@ func (t *TransportFactory) Supports(dsn string) bool { func (t *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { var optsConfig OptionsConfig if err := defaults.Set(&optsConfig); err != nil { - return nil, fmt.Errorf("kafka: set defaults: %w", err) + return nil, fmt.Errorf("set defaults: %w", err) } if err := yaml.Unmarshal(options, &optsConfig); err != nil { - return nil, fmt.Errorf("kafka: unmarshal options: %w", err) + return nil, fmt.Errorf("unmarshal options: %w", err) } tCfg := TransportConfig{ diff --git a/transport/kafka/factory_test.go b/transport/kafka/factory_test.go index c6f43e3..8a89e70 100644 --- a/transport/kafka/factory_test.go +++ b/transport/kafka/factory_test.go @@ -95,6 +95,5 @@ func TestTransportFactory_Create(t *testing.T) { transport, err := factory.Create(name, dsn, optionsBytes) require.Error(t, err) - assert.Contains(t, err.Error(), "kafka") assert.Nil(t, transport) } diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index 044f289..6b1c47c 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -23,7 +23,7 @@ type Transport struct { func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { u, err := url.Parse(cfg.DSN) if err != nil { - return nil, fmt.Errorf("kafka: failed to parse dsn: %w", err) + return nil, fmt.Errorf("failed to parse dsn: %w", err) } brokers := strings.Split(u.Host, ",") @@ -32,7 +32,7 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L if err != nil { logger.Error("failed to connect to Kafka brokers", "error", err) - return nil, fmt.Errorf("kafka: %w", err) + return nil, err } ser := serializer.NewSerializer(resolver) diff --git a/transport/manager.go b/transport/manager.go index a7b2785..4412921 100644 --- a/transport/manager.go +++ b/transport/manager.go @@ -53,6 +53,14 @@ func (m *Manager) Start(ctx context.Context, consumeOnly []string) { m.running = true for _, t := range m.transports { + if s, ok := t.(api.Setupable); ok { + if err := s.Setup(ctx); err != nil { + m.logger.ErrorContext(ctx, "failed to setup transport", "name", t.Name(), "error", err) + + continue + } + } + if !m.stringInSlice(t.Name(), consumeOnly) { continue } diff --git a/transport/redis/config.go b/transport/redis/config.go new file mode 100644 index 0000000..2725c18 --- /dev/null +++ b/transport/redis/config.go @@ -0,0 +1,14 @@ +package redis + +type TransportConfig struct { + Name string + DSN string + Options OptionsConfig +} + +type OptionsConfig struct { + AutoSetup bool `yaml:"auto_setup" default:"true"` + Stream string `yaml:"stream" default:"messages"` + Group string `yaml:"group" default:"default"` + Consumer string `yaml:"consumer" default:"consumer"` +} diff --git a/transport/redis/connection.go b/transport/redis/connection.go new file mode 100644 index 0000000..71b97c4 --- /dev/null +++ b/transport/redis/connection.go @@ -0,0 +1,48 @@ +package redis + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/redis/go-redis/v9" +) + +const connectionTimeout = 5 * time.Second + +type Connection struct { + client *redis.Client +} + +func NewConnection(dsn string) (*Connection, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, fmt.Errorf("failed to parse dsn: %w", err) + } + + opts := &redis.Options{ + Addr: u.Host, + } + + if u.User != nil { + if password, hasPassword := u.User.Password(); hasPassword { + opts.Password = password + } + } + + client := redis.NewClient(opts) + + ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) + defer cancel() + + if errPing := client.Ping(ctx).Err(); errPing != nil { + return nil, fmt.Errorf("ping failed: %w", errPing) + } + + return &Connection{client: client}, nil +} + +func (c *Connection) Client() *redis.Client { + return c.client +} diff --git a/transport/redis/consumer.go b/transport/redis/consumer.go new file mode 100644 index 0000000..ca8e270 --- /dev/null +++ b/transport/redis/consumer.go @@ -0,0 +1,139 @@ +package redis + +import ( + "context" + "errors" + "log/slog" + "strings" + "sync" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" +) + +const ( + defaultBatchSize = 10 +) + +type Consumer struct { + cfg TransportConfig + serializer api.Serializer + conn *Connection + logger *slog.Logger + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +func NewConsumer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) *Consumer { + ctx, cancel := context.WithCancel(context.Background()) + + return &Consumer{ + cfg: cfg, + serializer: ser, + conn: conn, + logger: logger, + ctx: ctx, + cancel: cancel, + } +} + +func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + group := c.cfg.Options.Group + stream := c.cfg.Options.Stream + + _ = c.conn.Client().XGroupCreateMkStream(ctx, stream, group, "$") + + c.wg.Add(1) + go func() { + defer c.wg.Done() + c.consumeLoop(handler) + }() + + <-ctx.Done() + c.cancel() + c.wg.Wait() + + return ctx.Err() +} + +func (c *Consumer) consumeLoop(handler func(context.Context, api.Envelope) error) { + rdb := c.conn.Client() + stream := c.cfg.Options.Stream + group := c.cfg.Options.Group + consumer := c.cfg.Options.Consumer + + for { + select { + case <-c.ctx.Done(): + return + default: + streams, err := rdb.XReadGroup(c.ctx, &redis.XReadGroupArgs{ + Group: group, + Consumer: consumer, + Streams: []string{stream, ">"}, + Count: defaultBatchSize, + Block: time.Second, + }).Result() + + if err != nil && !errors.Is(err, redis.Nil) { + c.logger.Error("XREADGROUP error", "error", err) + + continue + } + + for _, s := range streams { + for _, msg := range s.Messages { + c.handleMessage(msg, handler) + } + } + } + } +} + +func (c *Consumer) handleMessage(msg redis.XMessage, handler func(context.Context, api.Envelope) error) { + bodyRaw, ok := msg.Values["body"] + if !ok { + c.logger.Warn("missing 'body' field in message", "id", msg.ID) + + return + } + + bodyBytes, ok := bodyRaw.(string) + if !ok { + c.logger.Warn("invalid body format", "id", msg.ID) + + return + } + + headers := make(map[string]string) + for k, v := range msg.Values { + if strings.HasPrefix(k, "header_") { + if str, isString := v.(string); isString { + headers[strings.TrimPrefix(k, "header_")] = str + } + } + } + + env, errUnmarshal := c.serializer.Unmarshal([]byte(bodyBytes), headers) + if errUnmarshal != nil { + c.logger.Error("failed to unmarshal", "error", errUnmarshal) + + return + } + + env = env.WithStamp(stamps.ReceivedStamp{Transport: c.cfg.Name}) + + if errHandler := handler(c.ctx, env); errHandler != nil { + c.logger.Error("handler failed", "error", errHandler) + + return + } + + if err := c.conn.Client().XAck(c.ctx, c.cfg.Options.Stream, c.cfg.Options.Group, msg.ID).Err(); err != nil { + c.logger.Error("XACK failed", "id", msg.ID, "error", err) + } +} diff --git a/transport/redis/factory.go b/transport/redis/factory.go new file mode 100644 index 0000000..99a542c --- /dev/null +++ b/transport/redis/factory.go @@ -0,0 +1,47 @@ +package redis + +import ( + "fmt" + "log/slog" + "strings" + + "github.com/creasty/defaults" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/api" +) + +type TransportFactory struct { + logger *slog.Logger + resolver api.TypeResolver +} + +func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { + return &TransportFactory{ + logger: logger, + resolver: resolver, + } +} + +func (t *TransportFactory) Supports(dsn string) bool { + return strings.HasPrefix(dsn, "redis://") +} + +func (t *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { + var optsConfig OptionsConfig + if err := defaults.Set(&optsConfig); err != nil { + return nil, fmt.Errorf("set defaults: %w", err) + } + + if err := yaml.Unmarshal(options, &optsConfig); err != nil { + return nil, fmt.Errorf("unmarshal options: %w", err) + } + + tCfg := TransportConfig{ + Name: name, + DSN: dsn, + Options: optsConfig, + } + + return NewTransport(tCfg, t.resolver, t.logger) +} diff --git a/transport/redis/producer.go b/transport/redis/producer.go new file mode 100644 index 0000000..11e6125 --- /dev/null +++ b/transport/redis/producer.go @@ -0,0 +1,77 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "log/slog" + "regexp" + + "github.com/redis/go-redis/v9" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/envelope" + "github.com/gerfey/messenger/core/stamps" +) + +type Producer struct { + cfg TransportConfig + serializer api.Serializer + conn *Connection + logger *slog.Logger +} + +func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) *Producer { + return &Producer{ + cfg: cfg, + serializer: ser, + conn: conn, + logger: logger, + } +} + +func (p *Producer) Send(ctx context.Context, env api.Envelope) error { + payload, headers, err := p.serializer.Marshal(env) + if err != nil { + return fmt.Errorf("redis: marshal envelope failed: %w", err) + } + + data := map[string]any{ + "body": payload, + } + + for k, v := range headers { + data["header_"+k] = v + } + + stream := p.cfg.Options.Stream + if stream == "" { + return errors.New("redis: stream name is not configured") + } + + id := "*" + if stamp, ok := envelope.LastStampOf[stamps.MessageIDStamp](env); ok { + if p.isValidRedisStreamID(stamp.MessageID) { + id = stamp.MessageID + } + } + + _, err = p.conn.Client().XAdd(ctx, &redis.XAddArgs{ + ID: id, + Stream: stream, + Values: data, + }).Result() + if err != nil { + return fmt.Errorf("redis: XADD failed: %w", err) + } + + p.logger.DebugContext(ctx, "message sent to redis stream", + slog.String("stream", stream), + slog.String("message_type", fmt.Sprintf("%T", env.Message()))) + + return nil +} + +func (p *Producer) isValidRedisStreamID(id string) bool { + return regexp.MustCompile(`^\d+-\d+$`).MatchString(id) +} diff --git a/transport/redis/transport.go b/transport/redis/transport.go new file mode 100644 index 0000000..806d312 --- /dev/null +++ b/transport/redis/transport.go @@ -0,0 +1,75 @@ +package redis + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/serializer" +) + +type Transport struct { + cfg TransportConfig + producer *Producer + consumer *Consumer + serializer api.Serializer + logger *slog.Logger + conn *Connection +} + +func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { + conn, err := NewConnection(cfg.DSN) + if err != nil { + logger.Error("failed to connect", "error", err) + + return nil, err + } + + ser := serializer.NewSerializer(resolver) + + producer := NewProducer(cfg, ser, conn, logger) + consumer := NewConsumer(cfg, ser, conn, logger) + + return &Transport{ + cfg: cfg, + producer: producer, + consumer: consumer, + serializer: ser, + logger: logger, + conn: conn, + }, nil +} + +func (t *Transport) Name() string { + return t.cfg.Name +} + +func (t *Transport) Send(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} + +func (t *Transport) Receive(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + return t.consumer.Consume(ctx, handler) +} + +func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} + +func (t *Transport) Setup(ctx context.Context) error { + if !t.cfg.Options.AutoSetup { + return nil + } + + stream := t.cfg.Options.Stream + group := t.cfg.Options.Group + + _, err := t.conn.Client().XGroupCreateMkStream(ctx, stream, group, "$").Result() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return fmt.Errorf("failed to create consumer group: %w", err) + } + + return nil +} From 1883eb8c1bfcd6c5ae2b95f0527e323ce0b7f5ca Mon Sep 17 00:00:00 2001 From: Gerfey Date: Wed, 6 Aug 2025 12:25:49 +0700 Subject: [PATCH 12/17] feat: added support for serializers for transports with the possibility of configuration --- api/builder.go | 1 + api/serializer.go | 6 ++ api/transport.go | 2 +- builder/builder.go | 34 +++++++-- builder/builder_test.go | 15 ++-- config/config.go | 12 ++-- core/serializer/locator.go | 39 +++++++++++ {serializer => core/serializer}/serializer.go | 0 .../serializer}/serializer_test.go | 3 +- examples/serializer/handler/hello_handler.go | 20 ++++++ examples/serializer/message/hello.go | 9 +++ examples/serializer/messenger.yaml | 10 +++ examples/serializer/serializer.go | 69 +++++++++++++++++++ examples/serializer/serializer/serializer.go | 38 ++++++++++ go.mod | 4 +- go.sum | 4 ++ tests/helpers/messages.go | 2 +- transport/amqp/factory.go | 12 ++-- transport/amqp/factory_test.go | 14 ++-- transport/amqp/transport.go | 5 +- transport/chain.go | 8 ++- transport/chain_test.go | 46 ++++++++++--- transport/inmemory/factory.go | 10 ++- transport/inmemory/factory_test.go | 14 ++-- transport/kafka/factory.go | 12 ++-- transport/kafka/factory_test.go | 13 ++-- transport/kafka/transport.go | 5 +- transport/redis/factory.go | 12 ++-- transport/redis/transport.go | 5 +- transport/sync/factory.go | 13 ++-- transport/sync/factory_test.go | 18 ++--- 31 files changed, 345 insertions(+), 110 deletions(-) create mode 100644 core/serializer/locator.go rename {serializer => core/serializer}/serializer.go (100%) rename {serializer => core/serializer}/serializer_test.go (98%) create mode 100644 examples/serializer/handler/hello_handler.go create mode 100644 examples/serializer/message/hello.go create mode 100644 examples/serializer/messenger.yaml create mode 100644 examples/serializer/serializer.go create mode 100644 examples/serializer/serializer/serializer.go diff --git a/api/builder.go b/api/builder.go index de18d3a..8dcdc98 100644 --- a/api/builder.go +++ b/api/builder.go @@ -6,6 +6,7 @@ type Builder interface { RegisterStamp(any) RegisterListener(any, any) RegisterMiddleware(string, Middleware) + RegisterSerializer(string, Serializer) RegisterTransportFactory(TransportFactory) Build() (Messenger, error) } diff --git a/api/serializer.go b/api/serializer.go index 1ccf26c..74d16b4 100644 --- a/api/serializer.go +++ b/api/serializer.go @@ -4,3 +4,9 @@ type Serializer interface { Marshal(Envelope) ([]byte, map[string]string, error) Unmarshal(body []byte, headers map[string]string) (Envelope, error) } + +type SerializerLocator interface { + Register(string, Serializer) + GetAll() []Serializer + Get(name string) (Serializer, error) +} diff --git a/api/transport.go b/api/transport.go index b23d9bb..144e082 100644 --- a/api/transport.go +++ b/api/transport.go @@ -33,7 +33,7 @@ type SenderLocator interface { type TransportFactory interface { Supports(string) bool - Create(string, string, []byte) (Transport, error) + Create(string, string, []byte, Serializer) (Transport, error) } type RoutedMessage interface { diff --git a/builder/builder.go b/builder/builder.go index ed4fc79..672e72d 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -17,6 +17,7 @@ import ( "github.com/gerfey/messenger/core/middleware/implementation" "github.com/gerfey/messenger/core/retry" "github.com/gerfey/messenger/core/routing" + "github.com/gerfey/messenger/core/serializer" "github.com/gerfey/messenger/core/stamps" "github.com/gerfey/messenger/transport" "github.com/gerfey/messenger/transport/amqp" @@ -33,6 +34,7 @@ type Builder struct { handlersLocator api.HandlerLocator senderLocator api.SenderLocator middlewareLocator api.MiddlewareLocator + serializerLocator api.SerializerLocator busLocator api.BusLocator eventDispatcher api.EventDispatcher logger *slog.Logger @@ -42,13 +44,14 @@ func NewBuilder(cfg *config.MessengerConfig, logger *slog.Logger) api.Builder { resolver := NewResolver() busLocator := bus.NewLocator() + serializerLocator := serializer.NewSerializerLocator() tf := transport.NewFactoryChain( - amqp.NewTransportFactory(logger, resolver), - inmemory.NewTransportFactory(logger, resolver), - sync.NewTransportFactory(logger, busLocator), - kafka.NewTransportFactory(logger, resolver), - redis.NewTransportFactory(logger, resolver), + amqp.NewTransportFactory(logger), + inmemory.NewTransportFactory(logger), + sync.NewTransportFactory(busLocator), + kafka.NewTransportFactory(logger), + redis.NewTransportFactory(logger), ) return &Builder{ @@ -58,6 +61,7 @@ func NewBuilder(cfg *config.MessengerConfig, logger *slog.Logger) api.Builder { handlersLocator: handler.NewHandlerLocator(), senderLocator: transport.NewSenderLocator(), middlewareLocator: middleware.NewMiddlewareLocator(), + serializerLocator: serializerLocator, busLocator: busLocator, eventDispatcher: event.NewEventDispatcher(logger), logger: logger, @@ -80,6 +84,10 @@ func (b *Builder) RegisterHandler(handler any) error { return nil } +func (b *Builder) RegisterSerializer(name string, sz api.Serializer) { + b.serializerLocator.Register(name, sz) +} + func (b *Builder) RegisterMiddleware(name string, mw api.Middleware) { b.middlewareLocator.Register(name, mw) } @@ -101,6 +109,8 @@ func (b *Builder) RegisterListener(event any, listener any) { func (b *Builder) Build() (api.Messenger, error) { b.registerStamps() + b.serializerLocator.Register("default.transport.serializer", serializer.NewSerializer(b.resolver)) + if err := b.setupBuses(); err != nil { return nil, err } @@ -220,7 +230,17 @@ func (b *Builder) createTransports(manager *transport.Manager) (map[string]api.T b.createdSyncTransport(createdTransports) for name, tCfg := range b.cfg.Transports { - tr, err := b.transportFactory.CreateTransport(name, tCfg) + serializerName := tCfg.Serializer + if serializerName == "" { + serializerName = b.cfg.DefaultSerializer + } + + sz, err := b.serializerLocator.Get(serializerName) + if err != nil { + return nil, nil, fmt.Errorf("serializer %q not found for transport %q: %w", serializerName, name, err) + } + + tr, err := b.transportFactory.CreateTransport(name, tCfg, sz) if err != nil { return nil, nil, fmt.Errorf("failed to create transport '%s': %w", name, err) } @@ -283,7 +303,7 @@ func (b *Builder) createdSyncTransport(createdTransports map[string]api.Transpor DSN: "sync://", } - if syncTransport, err := b.transportFactory.CreateTransport("sync", cfg); err == nil { + if syncTransport, err := b.transportFactory.CreateTransport("sync", cfg, nil); err == nil { createdTransports["sync"] = syncTransport } } diff --git a/builder/builder_test.go b/builder/builder_test.go index 8ea6516..c6e9956 100644 --- a/builder/builder_test.go +++ b/builder/builder_test.go @@ -201,7 +201,8 @@ func TestBuilder_RegisterListener(t *testing.T) { func TestBuilder_RegisterTransportFactory(t *testing.T) { t.Run("register transport factory", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, @@ -256,7 +257,8 @@ func TestBuilder_RegisterTransportFactory(t *testing.T) { func TestBuilder_Build_Errors(t *testing.T) { t.Run("build fails with unknown message type in routing", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, @@ -319,7 +321,8 @@ func TestBuilder_Build_Errors(t *testing.T) { func TestBuilder_Build_Success(t *testing.T) { t.Run("build messenger with complete configuration", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{"test_middleware"}, @@ -368,7 +371,8 @@ func TestBuilder_Build_Success(t *testing.T) { t.Run("build messenger with retry configuration", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, @@ -404,7 +408,8 @@ func TestBuilder_Build_Success(t *testing.T) { t.Run("build messenger with multiple buses and transports", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, diff --git a/config/config.go b/config/config.go index 3093c35..9f4d879 100644 --- a/config/config.go +++ b/config/config.go @@ -11,11 +11,12 @@ const ( ) type MessengerConfig struct { - DefaultBus string `yaml:"default_bus" default:"default"` - FailureTransport string `yaml:"failure_transport"` - Buses map[string]BusConfig `yaml:"buses"` - Transports map[string]TransportConfig `yaml:"transports"` - Routing map[string]string `yaml:"routing"` + DefaultBus string `yaml:"default_bus" default:"default"` + DefaultSerializer string `yaml:"default_serializer" default:"default.transport.serializer"` + FailureTransport string `yaml:"failure_transport"` + Buses map[string]BusConfig `yaml:"buses"` + Transports map[string]TransportConfig `yaml:"transports"` + Routing map[string]string `yaml:"routing"` } type BusConfig struct { @@ -24,6 +25,7 @@ type BusConfig struct { type TransportConfig struct { DSN string `yaml:"dsn"` + Serializer string `yaml:"serializer"` RetryStrategy *RetryStrategyConfig `yaml:"retry_strategy"` Options map[string]any `yaml:"options"` } diff --git a/core/serializer/locator.go b/core/serializer/locator.go new file mode 100644 index 0000000..46605bb --- /dev/null +++ b/core/serializer/locator.go @@ -0,0 +1,39 @@ +package serializer + +import ( + "fmt" + + "github.com/gerfey/messenger/api" +) + +type Locator struct { + serializers map[string]api.Serializer +} + +func NewSerializerLocator() api.SerializerLocator { + return &Locator{ + serializers: make(map[string]api.Serializer), + } +} + +func (m *Locator) Register(name string, serializer api.Serializer) { + m.serializers[name] = serializer +} + +func (m *Locator) GetAll() []api.Serializer { + all := make([]api.Serializer, 0) + for _, serializer := range m.serializers { + all = append(all, serializer) + } + + return all +} + +func (m *Locator) Get(name string) (api.Serializer, error) { + mw, ok := m.serializers[name] + if !ok { + return nil, fmt.Errorf("no serializer with name %s found", name) + } + + return mw, nil +} diff --git a/serializer/serializer.go b/core/serializer/serializer.go similarity index 100% rename from serializer/serializer.go rename to core/serializer/serializer.go diff --git a/serializer/serializer_test.go b/core/serializer/serializer_test.go similarity index 98% rename from serializer/serializer_test.go rename to core/serializer/serializer_test.go index 681d956..988dd23 100644 --- a/serializer/serializer_test.go +++ b/core/serializer/serializer_test.go @@ -6,9 +6,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/builder" "github.com/gerfey/messenger/core/envelope" - "github.com/gerfey/messenger/serializer" "github.com/gerfey/messenger/tests/helpers" ) diff --git a/examples/serializer/handler/hello_handler.go b/examples/serializer/handler/hello_handler.go new file mode 100644 index 0000000..b2d42f6 --- /dev/null +++ b/examples/serializer/handler/hello_handler.go @@ -0,0 +1,20 @@ +package handler + +import ( + "context" + "fmt" + + "github.com/gerfey/messenger/examples/serializer/message" +) + +type ExampleHelloHandler struct{} + +func (u *ExampleHelloHandler) Handle(_ context.Context, msg *message.ExampleHelloMessage) error { + fmt.Printf("Handled: Text=%v\n", msg.Text) + + return nil +} + +func (u *ExampleHelloHandler) GetBusName() string { + return "default" +} diff --git a/examples/serializer/message/hello.go b/examples/serializer/message/hello.go new file mode 100644 index 0000000..a7877ee --- /dev/null +++ b/examples/serializer/message/hello.go @@ -0,0 +1,9 @@ +package message + +type ExampleHelloMessage struct { + Text string +} + +func (m *ExampleHelloMessage) RoutingKey() string { + return "test_routing_key" +} diff --git a/examples/serializer/messenger.yaml b/examples/serializer/messenger.yaml new file mode 100644 index 0000000..953745d --- /dev/null +++ b/examples/serializer/messenger.yaml @@ -0,0 +1,10 @@ +default_bus: default +default_serializer: default.transport.serializer + +buses: + default: ~ + +transports: + sync: + dsn: "sync://" + serializer: test.json diff --git a/examples/serializer/serializer.go b/examples/serializer/serializer.go new file mode 100644 index 0000000..568502d --- /dev/null +++ b/examples/serializer/serializer.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/examples/serializer/handler" + "github.com/gerfey/messenger/examples/serializer/message" + "github.com/gerfey/messenger/examples/serializer/serializer" +) + +const ( + waitDurationSeconds = 20 +) + +func main() { + ctx := context.Background() + + log := slog.Default() + + cfg, err := config.LoadConfig("./examples/serializer/messenger.yaml") + if err != nil { + log.Error("ERROR load config", "error", err) + + return + } + + b := builder.NewBuilder(cfg, log) + + _ = b.RegisterHandler(&handler.ExampleHelloHandler{}) + + b.RegisterSerializer("test.json", serializer.NewTestSerializer()) + + messenger, err := b.Build() + if err != nil { + log.Error("failed to build messenger", "error", err) + + return + } + + go func() { + if runErr := messenger.Run(ctx); runErr != nil { + log.Error("messenger run failed", "error", runErr) + + return + } + }() + + messengerBus, err := messenger.GetDefaultBus() + if err != nil { + log.Error("failed to get default bus", "error", err) + + return + } + + _, err = messengerBus.Dispatch(ctx, &message.ExampleHelloMessage{ + Text: "Hello World", + }) + if err != nil { + log.Error("failed to dispatch message", "error", err) + + return + } + + time.Sleep(waitDurationSeconds * time.Second) +} diff --git a/examples/serializer/serializer/serializer.go b/examples/serializer/serializer/serializer.go new file mode 100644 index 0000000..2654808 --- /dev/null +++ b/examples/serializer/serializer/serializer.go @@ -0,0 +1,38 @@ +package serializer + +import ( + "encoding/json" + "fmt" + "reflect" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/envelope" +) + +type TestSerializer struct{} + +func NewTestSerializer() api.Serializer { + return &TestSerializer{} +} + +func (s *TestSerializer) Marshal(env api.Envelope) ([]byte, map[string]string, error) { + msg := env.Message() + body, err := json.Marshal(msg) + if err != nil { + return nil, nil, err + } + + headers := map[string]string{ + "type": reflect.TypeOf(msg).String(), + } + + return body, headers, nil +} + +func (s *TestSerializer) Unmarshal(_ []byte, _ map[string]string) (api.Envelope, error) { + fmt.Println("TestJsonSerializer.Unmarshal") + + env := envelope.NewEnvelope("") + + return env, nil +} diff --git a/go.mod b/go.mod index 4d35b79..e3e3725 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.24 require ( github.com/creasty/defaults v1.8.0 + github.com/google/uuid v1.6.0 github.com/ilyakaznacheev/cleanenv v1.5.0 github.com/rabbitmq/amqp091-go v1.10.0 + github.com/redis/go-redis/v9 v9.11.0 github.com/segmentio/kafka-go v0.4.48 github.com/stretchr/testify v1.10.0 go.uber.org/mock v0.5.2 @@ -17,11 +19,9 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/google/uuid v1.6.0 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/redis/go-redis/v9 v9.11.0 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index 5008938..7b3f93e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= diff --git a/tests/helpers/messages.go b/tests/helpers/messages.go index 2efaaf1..67bb43e 100644 --- a/tests/helpers/messages.go +++ b/tests/helpers/messages.go @@ -220,7 +220,7 @@ func (f *TestTransportFactory) Supports(_ string) bool { return true } -func (f *TestTransportFactory) Create(_ string, _ string, _ []byte) (api.Transport, error) { +func (f *TestTransportFactory) Create(_ string, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { if f.CreateError != nil { return nil, f.CreateError } diff --git a/transport/amqp/factory.go b/transport/amqp/factory.go index ba98c49..4be2948 100644 --- a/transport/amqp/factory.go +++ b/transport/amqp/factory.go @@ -12,14 +12,12 @@ import ( ) type TransportFactory struct { - logger *slog.Logger - resolver api.TypeResolver + logger *slog.Logger } -func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { +func NewTransportFactory(logger *slog.Logger) api.TransportFactory { return &TransportFactory{ - logger: logger, - resolver: resolver, + logger: logger, } } @@ -27,7 +25,7 @@ func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "amqp://") } -func (f *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { +func (f *TransportFactory) Create(name string, dsn string, options []byte, ser api.Serializer) (api.Transport, error) { var opts OptionsConfig if err := defaults.Set(&opts); err != nil { return nil, fmt.Errorf("set defaults: %w", err) @@ -43,5 +41,5 @@ func (f *TransportFactory) Create(name string, dsn string, options []byte) (api. Options: opts, } - return NewTransport(cfg, f.resolver, f.logger) + return NewTransport(cfg, f.logger, ser) } diff --git a/transport/amqp/factory_test.go b/transport/amqp/factory_test.go index 7f65377..12a572d 100644 --- a/transport/amqp/factory_test.go +++ b/transport/amqp/factory_test.go @@ -9,6 +9,8 @@ import ( "go.uber.org/mock/gomock" "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/amqp" ) @@ -18,9 +20,8 @@ func TestNewTransportFactory(t *testing.T) { defer ctrl.Finish() logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := amqp.NewTransportFactory(logger, mockResolver) + factory := amqp.NewTransportFactory(logger) assert.NotNil(t, factory) assert.IsType(t, &amqp.TransportFactory{}, factory) @@ -65,8 +66,7 @@ func TestTransportFactory_Supports(t *testing.T) { defer ctrl.Finish() logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := amqp.NewTransportFactory(logger, mockResolver) + factory := amqp.NewTransportFactory(logger) got := factory.Supports(tt.dsn) assert.Equal(t, tt.want, got) @@ -80,7 +80,7 @@ func TestTransportFactory_Create(t *testing.T) { logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := amqp.NewTransportFactory(logger, mockResolver) + factory := amqp.NewTransportFactory(logger) name := "test-amqp" @@ -104,10 +104,12 @@ func TestTransportFactory_Create(t *testing.T) { }, } + ser := serializer.NewSerializer(mockResolver) + optionsBytes, err := yaml.Marshal(options) require.NoError(t, err) - _, err = factory.Create(name, dsn, optionsBytes) + _, err = factory.Create(name, dsn, optionsBytes, ser) require.Error(t, err) assert.Contains(t, err.Error(), "failed to connect") diff --git a/transport/amqp/transport.go b/transport/amqp/transport.go index fd93fab..a2aa875 100644 --- a/transport/amqp/transport.go +++ b/transport/amqp/transport.go @@ -7,7 +7,6 @@ import ( "reflect" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/serializer" ) type Transport struct { @@ -20,7 +19,7 @@ type Transport struct { logger *slog.Logger } -func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { +func NewTransport(cfg TransportConfig, logger *slog.Logger, ser api.Serializer) (api.Transport, error) { conn, err := NewConnection(cfg.DSN) if err != nil { logger.Error("failed to create AMQP connection", "dsn", cfg.DSN, "error", err) @@ -28,8 +27,6 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L return nil, err } - ser := serializer.NewSerializer(resolver) - pub := NewPublisher(conn, cfg, ser) cons := NewConsumer(conn, cfg, ser) ret := NewRetry(conn, cfg, ser) diff --git a/transport/chain.go b/transport/chain.go index 5fbaf74..dfb18ee 100644 --- a/transport/chain.go +++ b/transport/chain.go @@ -17,7 +17,11 @@ func NewFactoryChain(factories ...api.TransportFactory) *FactoryChain { return &FactoryChain{factories: factories} } -func (c *FactoryChain) CreateTransport(name string, config config.TransportConfig) (api.Transport, error) { +func (c *FactoryChain) CreateTransport( + name string, + config config.TransportConfig, + sz api.Serializer, +) (api.Transport, error) { for _, factory := range c.factories { if factory.Supports(config.DSN) { rawOptions, errOptions := yaml.Marshal(config.Options) @@ -25,7 +29,7 @@ func (c *FactoryChain) CreateTransport(name string, config config.TransportConfi return nil, fmt.Errorf("%s: marshal options map: %w", name, errOptions) } - return factory.Create(name, config.DSN, rawOptions) + return factory.Create(name, config.DSN, rawOptions, sz) } } diff --git a/transport/chain_test.go b/transport/chain_test.go index 6c72ead..13e8a98 100644 --- a/transport/chain_test.go +++ b/transport/chain_test.go @@ -6,6 +6,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/transport" "github.com/gerfey/messenger/api" @@ -23,7 +26,7 @@ func (f *mockFactory) Supports(dsn string) bool { return dsn == f.supportedDSN } -func (f *mockFactory) Create(name, _ string, _ []byte) (api.Transport, error) { +func (f *mockFactory) Create(name, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { if f.createError != nil { return nil, f.createError } @@ -84,7 +87,10 @@ func TestFactoryChain_CreateTransport(t *testing.T) { Options: map[string]any{}, } - tr, err := chain.CreateTransport("test-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("test-transport", transportConfig, ser) require.NoError(t, err) assert.Same(t, expectedTransport, tr) @@ -106,7 +112,10 @@ func TestFactoryChain_CreateTransport(t *testing.T) { Options: map[string]any{}, } - tr, err := chain.CreateTransport("first-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("first-transport", transportConfig, ser) require.NoError(t, err) assert.Same(t, expectedTransport, tr) @@ -122,7 +131,10 @@ func TestFactoryChain_CreateTransport(t *testing.T) { Options: map[string]any{}, } - tr, err := chain.CreateTransport("unknown-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("unknown-transport", transportConfig, ser) require.Error(t, err) assert.Nil(t, tr) @@ -144,7 +156,10 @@ func TestFactoryChain_CreateTransport(t *testing.T) { Options: map[string]any{}, } - tr, err := chain.CreateTransport("error-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("error-transport", transportConfig, ser) require.Error(t, err) assert.Same(t, expectedError, err) @@ -159,7 +174,10 @@ func TestFactoryChain_CreateTransport(t *testing.T) { Options: map[string]any{}, } - tr, err := chain.CreateTransport("test-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("test-transport", transportConfig, ser) require.Error(t, err) assert.Nil(t, tr) @@ -182,7 +200,10 @@ func TestFactoryChain_CreateTransport(t *testing.T) { Options: map[string]any{}, } - tr, err := chain.CreateTransport("second-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("second-transport", transportConfig, ser) require.NoError(t, err) assert.Same(t, expectedTransport, tr) @@ -235,22 +256,25 @@ func TestFactoryChain_Integration(t *testing.T) { memoryConfig := config.TransportConfig{DSN: "memory://"} unknownConfig := config.TransportConfig{DSN: "unknown://localhost"} - amqpTransport, err := chain.CreateTransport("amqp", amqpConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + amqpTransport, err := chain.CreateTransport("amqp", amqpConfig, ser) require.NoError(t, err) require.NotNil(t, amqpTransport) assert.Equal(t, "amqp", amqpTransport.Name()) - redisTransport, err := chain.CreateTransport("redis", redisConfig) + redisTransport, err := chain.CreateTransport("redis", redisConfig, ser) require.NoError(t, err) require.NotNil(t, redisTransport) assert.Equal(t, "redis", redisTransport.Name()) - memoryTransport, err := chain.CreateTransport("memory", memoryConfig) + memoryTransport, err := chain.CreateTransport("memory", memoryConfig, ser) require.NoError(t, err) require.NotNil(t, memoryTransport) assert.Equal(t, "memory", memoryTransport.Name()) - unknownTransport, err := chain.CreateTransport("unknown", unknownConfig) + unknownTransport, err := chain.CreateTransport("unknown", unknownConfig, ser) require.Error(t, err) assert.Nil(t, unknownTransport) diff --git a/transport/inmemory/factory.go b/transport/inmemory/factory.go index 3dd680d..0af5f98 100644 --- a/transport/inmemory/factory.go +++ b/transport/inmemory/factory.go @@ -8,14 +8,12 @@ import ( ) type TransportFactory struct { - logger *slog.Logger - resolver api.TypeResolver + logger *slog.Logger } -func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { +func NewTransportFactory(logger *slog.Logger) api.TransportFactory { return &TransportFactory{ - logger: logger, - resolver: resolver, + logger: logger, } } @@ -23,6 +21,6 @@ func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "in-memory://") } -func (f *TransportFactory) Create(name string, _ string, _ []byte) (api.Transport, error) { +func (f *TransportFactory) Create(name string, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { return NewTransport(name), nil } diff --git a/transport/inmemory/factory_test.go b/transport/inmemory/factory_test.go index 661ad88..cacfe6d 100644 --- a/transport/inmemory/factory_test.go +++ b/transport/inmemory/factory_test.go @@ -9,6 +9,8 @@ import ( "go.uber.org/mock/gomock" "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/inmemory" ) @@ -18,9 +20,7 @@ func TestNewTransportFactory(t *testing.T) { defer ctrl.Finish() logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - - factory := inmemory.NewTransportFactory(logger, mockResolver) + factory := inmemory.NewTransportFactory(logger) assert.NotNil(t, factory) assert.IsType(t, &inmemory.TransportFactory{}, factory) @@ -31,8 +31,7 @@ func TestTransportFactory_Supports(t *testing.T) { defer ctrl.Finish() logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := inmemory.NewTransportFactory(logger, mockResolver) + factory := inmemory.NewTransportFactory(logger) testCases := []struct { name string @@ -75,16 +74,17 @@ func TestTransportFactory_Create(t *testing.T) { logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := inmemory.NewTransportFactory(logger, mockResolver) + factory := inmemory.NewTransportFactory(logger) name := "test-inmemory" dsn := "in-memory://test" options := map[string]any{} + ser := serializer.NewSerializer(mockResolver) optionsBytes, err := yaml.Marshal(options) require.NoError(t, err) - transport, err := factory.Create(name, dsn, optionsBytes) + transport, err := factory.Create(name, dsn, optionsBytes, ser) require.NoError(t, err) assert.NotNil(t, transport) diff --git a/transport/kafka/factory.go b/transport/kafka/factory.go index 682db81..94cd7ce 100644 --- a/transport/kafka/factory.go +++ b/transport/kafka/factory.go @@ -12,14 +12,12 @@ import ( ) type TransportFactory struct { - logger *slog.Logger - resolver api.TypeResolver + logger *slog.Logger } -func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { +func NewTransportFactory(logger *slog.Logger) api.TransportFactory { return &TransportFactory{ - logger: logger, - resolver: resolver, + logger: logger, } } @@ -27,7 +25,7 @@ func (t *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "kafka://") } -func (t *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { +func (t *TransportFactory) Create(name string, dsn string, options []byte, ser api.Serializer) (api.Transport, error) { var optsConfig OptionsConfig if err := defaults.Set(&optsConfig); err != nil { return nil, fmt.Errorf("set defaults: %w", err) @@ -43,5 +41,5 @@ func (t *TransportFactory) Create(name string, dsn string, options []byte) (api. Options: optsConfig, } - return NewTransport(tCfg, t.resolver, t.logger) + return NewTransport(tCfg, t.logger, ser) } diff --git a/transport/kafka/factory_test.go b/transport/kafka/factory_test.go index 8a89e70..b05f454 100644 --- a/transport/kafka/factory_test.go +++ b/transport/kafka/factory_test.go @@ -9,6 +9,8 @@ import ( "go.uber.org/mock/gomock" "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/kafka" ) @@ -18,9 +20,8 @@ func TestNewTransportFactory(t *testing.T) { defer ctrl.Finish() logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := kafka.NewTransportFactory(logger, mockResolver) + factory := kafka.NewTransportFactory(logger) assert.NotNil(t, factory) assert.IsType(t, &kafka.TransportFactory{}, factory) @@ -65,8 +66,7 @@ func TestTransportFactory_Supports(t *testing.T) { defer ctrl.Finish() logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := kafka.NewTransportFactory(logger, mockResolver) + factory := kafka.NewTransportFactory(logger) result := factory.Supports(tc.dsn) assert.Equal(t, tc.expected, result) @@ -80,7 +80,7 @@ func TestTransportFactory_Create(t *testing.T) { logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := kafka.NewTransportFactory(logger, mockResolver) + factory := kafka.NewTransportFactory(logger) name := "test-kafka" dsn := "kafka://non-existent-host:9092" @@ -88,11 +88,12 @@ func TestTransportFactory_Create(t *testing.T) { Topics: []string{"test-topic"}, Group: "test-group", } + ser := serializer.NewSerializer(mockResolver) optionsBytes, err := yaml.Marshal(options) require.NoError(t, err) - transport, err := factory.Create(name, dsn, optionsBytes) + transport, err := factory.Create(name, dsn, optionsBytes, ser) require.Error(t, err) assert.Nil(t, transport) diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index 6b1c47c..3aa2c5f 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/serializer" ) type Transport struct { @@ -20,7 +19,7 @@ type Transport struct { conn *Connection } -func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { +func NewTransport(cfg TransportConfig, logger *slog.Logger, ser api.Serializer) (api.Transport, error) { u, err := url.Parse(cfg.DSN) if err != nil { return nil, fmt.Errorf("failed to parse dsn: %w", err) @@ -35,8 +34,6 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L return nil, err } - ser := serializer.NewSerializer(resolver) - producer, err := NewProducer(cfg, ser, conn, logger) if err != nil { return nil, fmt.Errorf("failed to create kafka producer: %w", err) diff --git a/transport/redis/factory.go b/transport/redis/factory.go index 99a542c..95c390a 100644 --- a/transport/redis/factory.go +++ b/transport/redis/factory.go @@ -12,14 +12,12 @@ import ( ) type TransportFactory struct { - logger *slog.Logger - resolver api.TypeResolver + logger *slog.Logger } -func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { +func NewTransportFactory(logger *slog.Logger) api.TransportFactory { return &TransportFactory{ - logger: logger, - resolver: resolver, + logger: logger, } } @@ -27,7 +25,7 @@ func (t *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "redis://") } -func (t *TransportFactory) Create(name string, dsn string, options []byte) (api.Transport, error) { +func (t *TransportFactory) Create(name string, dsn string, options []byte, ser api.Serializer) (api.Transport, error) { var optsConfig OptionsConfig if err := defaults.Set(&optsConfig); err != nil { return nil, fmt.Errorf("set defaults: %w", err) @@ -43,5 +41,5 @@ func (t *TransportFactory) Create(name string, dsn string, options []byte) (api. Options: optsConfig, } - return NewTransport(tCfg, t.resolver, t.logger) + return NewTransport(tCfg, t.logger, ser) } diff --git a/transport/redis/transport.go b/transport/redis/transport.go index 806d312..149da75 100644 --- a/transport/redis/transport.go +++ b/transport/redis/transport.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/serializer" ) type Transport struct { @@ -19,7 +18,7 @@ type Transport struct { conn *Connection } -func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { +func NewTransport(cfg TransportConfig, logger *slog.Logger, ser api.Serializer) (api.Transport, error) { conn, err := NewConnection(cfg.DSN) if err != nil { logger.Error("failed to connect", "error", err) @@ -27,8 +26,6 @@ func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.L return nil, err } - ser := serializer.NewSerializer(resolver) - producer := NewProducer(cfg, ser, conn, logger) consumer := NewConsumer(cfg, ser, conn, logger) diff --git a/transport/sync/factory.go b/transport/sync/factory.go index 63e8a59..9ae0c6f 100644 --- a/transport/sync/factory.go +++ b/transport/sync/factory.go @@ -1,28 +1,25 @@ package sync import ( - "log/slog" "strings" "github.com/gerfey/messenger/api" ) -type Factory struct { - logger *slog.Logger +type TransportFactory struct { locator api.BusLocator } -func NewTransportFactory(logger *slog.Logger, locator api.BusLocator) api.TransportFactory { - return &Factory{ - logger: logger, +func NewTransportFactory(locator api.BusLocator) api.TransportFactory { + return &TransportFactory{ locator: locator, } } -func (f *Factory) Supports(dsn string) bool { +func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "sync://") } -func (f *Factory) Create(_ string, _ string, _ []byte) (api.Transport, error) { +func (f *TransportFactory) Create(_ string, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { return NewTransport(f.locator), nil } diff --git a/transport/sync/factory_test.go b/transport/sync/factory_test.go index 4f0b400..09f1255 100644 --- a/transport/sync/factory_test.go +++ b/transport/sync/factory_test.go @@ -1,7 +1,6 @@ package sync_test import ( - "log/slog" "testing" "github.com/stretchr/testify/assert" @@ -9,6 +8,8 @@ import ( "go.uber.org/mock/gomock" "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/sync" ) @@ -17,22 +18,20 @@ func TestNewTransportFactory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockLocator := mocks.NewMockBusLocator(ctrl) - factory := sync.NewTransportFactory(logger, mockLocator) + factory := sync.NewTransportFactory(mockLocator) assert.NotNil(t, factory) - assert.IsType(t, &sync.Factory{}, factory) + assert.IsType(t, &sync.TransportFactory{}, factory) } func TestFactory_Supports(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockLocator := mocks.NewMockBusLocator(ctrl) - factory := sync.NewTransportFactory(logger, mockLocator) + factory := sync.NewTransportFactory(mockLocator) testCases := []struct { name string @@ -73,18 +72,19 @@ func TestFactory_Create(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockLocator := mocks.NewMockBusLocator(ctrl) - factory := sync.NewTransportFactory(logger, mockLocator) + factory := sync.NewTransportFactory(mockLocator) name := "test-sync" dsn := "sync://" options := map[string]any{} + mockResolver := mocks.NewMockTypeResolver(ctrl) + ser := serializer.NewSerializer(mockResolver) optionsBytes, err := yaml.Marshal(options) require.NoError(t, err) - transport, err := factory.Create(name, dsn, optionsBytes) + transport, err := factory.Create(name, dsn, optionsBytes, ser) require.NoError(t, err) assert.NotNil(t, transport) From 7c5837043a8d785034e771020d7ad5afbada4801 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Wed, 6 Aug 2025 14:33:58 +0700 Subject: [PATCH 13/17] docs: Added AMQP transport benchmarks and updated documentation for v0.8.0 --- README.md | 12 ++- README.ru.md | 10 +- docs/benchmark/AMQP-Benchmark.md | 45 ++++++++ transport/amqp/benchmark_test.go | 178 +++++++++++++++++++++++++++++++ 4 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 docs/benchmark/AMQP-Benchmark.md create mode 100644 transport/amqp/benchmark_test.go diff --git a/README.md b/README.md index c8631cc..55501e3 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,14 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/Gerfey/messenger.svg)](https://pkg.go.dev/github.com/Gerfey/messenger) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -> `v0.7.0` is a pre-release version — feel free to test and report issues! +> `v0.8.0` is a pre-release version — feel free to test and report issues! > Full documentation available in the [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) 🇷🇺 [Русская версия](README.ru.md) ## Features -- **Multiple Transports**: AMQP (RabbitMQ), In-Memory (sync) +- **Multiple Transports**: AMQP (RabbitMQ), Kafka, Redis (Stream), In-Memory (sync) - **Middleware Chain**: Extensible middleware system for message processing - **Event-Driven**: Built-in event dispatcher for lifecycle hooks - **Retry Mechanism**: Configurable retry strategies with exponential backoff @@ -25,7 +25,7 @@ ## Installation > Requires Go 1.24+ ```bash -go get github.com/gerfey/messenger@v0.7.0 +go get github.com/gerfey/messenger@v0.8.0 ``` ## Quick Start @@ -93,6 +93,10 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) > See [Usage Scenarios](https://github.com/Gerfey/messenger/wiki/Usage-Scenarios) for commands, queries, return values and advanced use-cases. +## Efficiency + +- AMQP (RabbitMQ): [AMQP Benchmark Report](docs/benchmark/AMQP-Benchmark.md) + ## Contributing 1. Fork the repository @@ -101,7 +105,7 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) 4. Push to the branch (`git push origin feature/amazing-feature`) 5. Open a Pull Request -## ⚖License +## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/README.ru.md b/README.ru.md index 7718829..9b85781 100644 --- a/README.ru.md +++ b/README.ru.md @@ -7,7 +7,7 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/Gerfey/messenger.svg)](https://pkg.go.dev/github.com/Gerfey/messenger) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -> Версия `v0.7.0` — это пре-релиз. Тестируйте и сообщайте о багах! +> Версия `v0.8.0` — это пре-релиз. Тестируйте и сообщайте о багах! > Полная документация доступна на [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) @@ -16,7 +16,7 @@ --- ## Возможности -- **Множественные транспорты**: AMQP (RabbitMQ), In-Memory (`sync`) +- **Множественные транспорты**: AMQP (RabbitMQ), Kafka, Redis (Stream), In-Memory (sync) - **Цепочка middleware**: Расширяемая система промежуточной обработки - **Событийный движок**: Встроенный dispatcher событий жизненного цикла - **Механизм повторов**: Настраиваемые стратегии ретраев с поддержкой DLQ @@ -27,7 +27,7 @@ ## Установка > Требуется Go версии **1.24+** ```bash -go get github.com/gerfey/messenger@v0.7.0 +go get github.com/gerfey/messenger@v0.8.0 ``` ## Быстрый старт @@ -95,6 +95,10 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) > Смотри [Сценарии использования](https://github.com/Gerfey/messenger/wiki/Сценарии-использования). +## Производительность + +- AMQP (RabbitMQ): [AMQP Benchmark Report](docs/benchmark/AMQP-Benchmark.md) + ## Как внести вклад 1. Форкните репозиторий diff --git a/docs/benchmark/AMQP-Benchmark.md b/docs/benchmark/AMQP-Benchmark.md new file mode 100644 index 0000000..595ea2b --- /dev/null +++ b/docs/benchmark/AMQP-Benchmark.md @@ -0,0 +1,45 @@ +# AMQP Benchmark Report + +Current performance results of the AMQP transport in Messenger (`v0.8.0`), tested with `RabbitMQ` using the `amqp091-go` client. + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +|------------------------------|---------------|---------------------|----------------|-----------| +| `BenchmarkAMQPSend` | 465,508 | ~2,148 | 13,822 | 267 | +| `BenchmarkAMQPConcurrentSend`| 296,922 | ~3,368 | 13,829 | 266 | + +*Concurrent sending is ~36% faster with 100B messages.* + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +|------------------|---------------|---------------------|----------------|-----------| +| 100 B | 472,831 | ~2,115 | 13,826 | 267 | +| 1 KB | 557,170 | ~1,794 | 19,874 | 267 | +| 10 KB | 704,831 | ~1,418 | 82,185 | 269 | +| 100 KB | 1,788,004 | ~559 | 726,393 | 276 | + +*As the payload size increases, throughput decreases and memory pressure on GC grows, as expected.* + +--- + +## Allocations: `pprof` Analysis +> Collected using `go test -bench=BenchmarkAMQP -benchmem -memprofile mem.out`, analyzed via `pprof`. + +(pprof) top + +- encoding/json: ~20% +- amqp091-go (sendOpen, Ack, readLongstr): ~25% +- Envelope.WithStamp: 5.75% +- Middleware chain: ~5% + +--- + +## Summary + +- AMQP transport in Messenger demonstrates stable and predictable performance +- Memory and allocation optimization opportunities are being addressed in upcoming versions + diff --git a/transport/amqp/benchmark_test.go b/transport/amqp/benchmark_test.go new file mode 100644 index 0000000..198fc8e --- /dev/null +++ b/transport/amqp/benchmark_test.go @@ -0,0 +1,178 @@ +package amqp_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/config" +) + +const ( + benchmarkAMQPDSN = "amqp://guest:guest@localhost:5672/" + benchmarkTimeout = 60 * time.Second +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func (m *BenchmarkMessage) RoutingKey() string { + return "benchmark_routing_key" +} + +type BenchmarkHandler struct { + processedCount *int64 + wg *sync.WaitGroup +} + +func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { + atomic.AddInt64(h.processedCount, 1) + if h.wg != nil { + h.wg.Done() + } + + return nil +} + +func (h *BenchmarkHandler) GetBusName() string { + return "default" +} + +func setupAMQPMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { + b.Helper() + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "amqp": { + DSN: benchmarkAMQPDSN, + Serializer: "default.transport.serializer", + Options: map[string]any{ + "auto_setup": true, + "exchange": map[string]any{ + "name": "benchmark_exchange", + "type": "topic", + }, + "queues": map[string]any{ + "benchmark_queue": map[string]any{ + "binding_keys": []string{"benchmark_routing_key"}, + }, + }, + }, + }, + }, + Routing: map[string]string{ + "*amqp_test.BenchmarkMessage": "amqp", + }, + } + + processedCount := int64(0) + var wg *sync.WaitGroup + if withWaitGroup { + wg = &sync.WaitGroup{} + } + + handler := &BenchmarkHandler{ + processedCount: &processedCount, + wg: wg, + } + + builderInstance := builder.NewBuilder(cfg, logger) + if err := builderInstance.RegisterHandler(handler); err != nil { + b.Fatalf("Register handler failed: %v", err) + } + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) + go func() { + defer cancel() + if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { + b.Logf("Messenger run error: %v", runErr) + } + }() + + time.Sleep(2 * time.Second) + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkAMQPSend(b *testing.B) { + bus := setupAMQPMessenger(b, false) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkAMQPConcurrentSend(b *testing.B) { + bus := setupAMQPMessenger(b, false) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkAMQPMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupAMQPMessenger(b, false) + dispatchMessages(b, bus, size, false) + }) + } +} From 957fe4b15dd40ab7f03ac411d9ca966ff090d548 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Thu, 7 Aug 2025 18:08:45 +0700 Subject: [PATCH 14/17] feat: the Kafka producer has been optimized --- examples/kafka_transport/messenger.yaml | 59 ++++---- transport/amqp/benchmark_test.go | 18 +-- transport/kafka/benchmark_test.go | 173 ++++++++++++++++++++++++ transport/kafka/config.go | 45 ++++-- transport/kafka/connection.go | 17 ++- transport/kafka/consumer.go | 52 ++++--- transport/kafka/producer.go | 99 ++++++++++---- transport/kafka/transport.go | 4 + transport/redis/benchmark_test.go | 172 +++++++++++++++++++++++ transport/sync/benchmark_test.go | 166 +++++++++++++++++++++++ 10 files changed, 692 insertions(+), 113 deletions(-) create mode 100644 transport/kafka/benchmark_test.go create mode 100644 transport/redis/benchmark_test.go create mode 100644 transport/sync/benchmark_test.go diff --git a/examples/kafka_transport/messenger.yaml b/examples/kafka_transport/messenger.yaml index db0e13d..892b87e 100644 --- a/examples/kafka_transport/messenger.yaml +++ b/examples/kafka_transport/messenger.yaml @@ -7,40 +7,41 @@ buses: transports: kafka: dsn: "kafka://localhost:29092/" - retry_strategy: - max_retries: 5 - delay: 500ms - multiplier: 2 - max_delay: 5s + serializer: default.transport.serializer options: topics: - my-topic group: my-group - offset_config: - type: earliest - commit: - strategy: batch - interval: 500ms - batch_size: 10 - pool: - size: 3 - min_size: 2 - max_size: 10 - dynamic: true - rebalance: - strategy: roundrobin - key: - strategy: message_id - failed_messages: - dsn: "amqp://guest:guest@localhost:5672/" - options: - auto_setup: true - exchange: - name: failed_exchange - type: fanout - queues: - failed_messages_queue: ~ + producer: + async: true + required_acks: -1 # -1: all replicas (strongest durability), 0: no ack, 1: leader only + batch_size: 1000 + batch_timeout: 20ms + write_timeout: 10s + read_timeout: 10s + balancer: least_bytes # least_bytes, hash, round_robin + + consumer: + offset: + type: earliest # earliest, latest, specific + value: 0 # used only for "specific" + commit: + strategy: batch # auto, batch, deferred + interval: 500ms + batch_size: 10 + rebalance: + strategy: roundrobin # roundrobin, range + pool: + size: 3 + min_size: 2 + max_size: 10 + dynamic: true + session_timeout: 10s + heartbeat_interval: 2s + + key: + strategy: message_id # none / message_id routing: message.ExampleHelloMessage: kafka diff --git a/transport/amqp/benchmark_test.go b/transport/amqp/benchmark_test.go index 198fc8e..5d3197f 100644 --- a/transport/amqp/benchmark_test.go +++ b/transport/amqp/benchmark_test.go @@ -17,7 +17,7 @@ import ( ) const ( - benchmarkAMQPDSN = "amqp://guest:guest@localhost:5672/" + benchmarkDSN = "amqp://guest:guest@localhost:5672/" benchmarkTimeout = 60 * time.Second ) @@ -49,7 +49,7 @@ func (h *BenchmarkHandler) GetBusName() string { return "default" } -func setupAMQPMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { +func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { b.Helper() logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ @@ -63,7 +63,7 @@ func setupAMQPMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { }, Transports: map[string]config.TransportConfig{ "amqp": { - DSN: benchmarkAMQPDSN, + DSN: benchmarkDSN, Serializer: "default.transport.serializer", Options: map[string]any{ "auto_setup": true, @@ -157,21 +157,21 @@ func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) } } -func BenchmarkAMQPSend(b *testing.B) { - bus := setupAMQPMessenger(b, false) +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b, false) dispatchMessages(b, bus, 100, false) } -func BenchmarkAMQPConcurrentSend(b *testing.B) { - bus := setupAMQPMessenger(b, false) +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b, false) dispatchMessages(b, bus, 100, true) } -func BenchmarkAMQPMessageSizes(b *testing.B) { +func BenchmarkMessageSizes(b *testing.B) { sizes := []int{100, 1024, 10240, 102400} for _, size := range sizes { b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { - bus := setupAMQPMessenger(b, false) + bus := setupMessenger(b, false) dispatchMessages(b, bus, size, false) }) } diff --git a/transport/kafka/benchmark_test.go b/transport/kafka/benchmark_test.go new file mode 100644 index 0000000..2bb06c8 --- /dev/null +++ b/transport/kafka/benchmark_test.go @@ -0,0 +1,173 @@ +package kafka_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/config" +) + +const ( + benchmarkDSN = "kafka://localhost:29092/" + benchmarkTimeout = 60 * time.Second +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func (m *BenchmarkMessage) RoutingKey() string { + return "benchmark_routing_key" +} + +type BenchmarkHandler struct { + processedCount *int64 + wg *sync.WaitGroup +} + +func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { + atomic.AddInt64(h.processedCount, 1) + if h.wg != nil { + h.wg.Done() + } + + return nil +} + +func (h *BenchmarkHandler) GetBusName() string { + return "default" +} + +func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { + b.Helper() + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "kafka": { + DSN: benchmarkDSN, + Serializer: "default.transport.serializer", + Options: map[string]any{ + "topics": []string{"benchmark_topic"}, + "group": "benchmark_group", + "producer": map[string]any{ + "async": false, + }, + }, + }, + }, + Routing: map[string]string{ + "*kafka_test.BenchmarkMessage": "kafka", + }, + } + + processedCount := int64(0) + var wg *sync.WaitGroup + if withWaitGroup { + wg = &sync.WaitGroup{} + } + + handler := &BenchmarkHandler{ + processedCount: &processedCount, + wg: wg, + } + + builderInstance := builder.NewBuilder(cfg, logger) + if err := builderInstance.RegisterHandler(handler); err != nil { + b.Fatalf("Register handler failed: %v", err) + } + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) + go func() { + defer cancel() + if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { + b.Logf("Messenger run error: %v", runErr) + } + }() + + time.Sleep(2 * time.Second) + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, size, false) + }) + } +} diff --git a/transport/kafka/config.go b/transport/kafka/config.go index c18971a..32c1392 100644 --- a/transport/kafka/config.go +++ b/transport/kafka/config.go @@ -9,13 +9,30 @@ type TransportConfig struct { } type OptionsConfig struct { - Topics []string `yaml:"topics,omitempty"` - Group string `yaml:"group" default:"group"` - OffsetConfig OffsetConfig `yaml:"offset_config"` - Commit CommitConfig `yaml:"commit"` - Pool PoolConfig `yaml:"pool"` - Rebalance RebalanceConfig `yaml:"rebalance"` - Key KeyConfig `yaml:"key"` + Topics []string `yaml:"topics,omitempty"` + Group string `yaml:"group" default:"default-group"` + Producer ProducerOptionsConfig `yaml:"producer"` + Consumer ConsumerOptionsConfig `yaml:"consumer"` + Key KeyConfig `yaml:"key"` +} + +type ProducerOptionsConfig struct { + Async bool `yaml:"async" default:"false"` + RequiredAcks int `yaml:"required_acks" default:"1"` // 0, 1, -1 (all) + BatchSize int `yaml:"batch_size" default:"1000"` + BatchTimeout time.Duration `yaml:"batch_timeout" default:"20ms"` + WriteTimeout time.Duration `yaml:"write_timeout" default:"10s"` + ReadTimeout time.Duration `yaml:"read_timeout" default:"10s"` + Balancer string `yaml:"balancer" default:"least_bytes"` // least_bytes, hash, round_robin +} + +type ConsumerOptionsConfig struct { + OffsetConfig OffsetConfig `yaml:"offset"` + Commit CommitConfig `yaml:"commit"` + Pool PoolConfig `yaml:"pool"` + Rebalance RebalanceConfig `yaml:"rebalance"` + SessionTimeout time.Duration `yaml:"session_timeout" default:"10s"` + HeartbeatInterval time.Duration `yaml:"heartbeat_interval" default:"2s"` } type OffsetConfig struct { @@ -24,16 +41,16 @@ type OffsetConfig struct { } type CommitConfig struct { - Strategy string `yaml:"strategy" default:"auto"` - Interval time.Duration `yaml:"interval" default:"1s"` - BatchSize int `yaml:"batch_size" default:"100"` + Strategy string `yaml:"strategy" default:"batch"` // auto, manual, batch, deferred + Interval time.Duration `yaml:"interval" default:"500ms"` // only for batch + BatchSize int `yaml:"batch_size" default:"10"` // only for batch } type PoolConfig struct { - Size int `yaml:"size" default:"10"` - MinSize int `yaml:"min_size" default:"5"` - MaxSize int `yaml:"max_size" default:"50"` - Dynamic bool `yaml:"dynamic" default:"false"` + Size int `yaml:"size" default:"3"` + MinSize int `yaml:"min_size" default:"2"` + MaxSize int `yaml:"max_size" default:"10"` + Dynamic bool `yaml:"dynamic" default:"true"` } type RebalanceConfig struct { diff --git a/transport/kafka/connection.go b/transport/kafka/connection.go index 08c9d61..7600b4a 100644 --- a/transport/kafka/connection.go +++ b/transport/kafka/connection.go @@ -58,12 +58,21 @@ func (c *Connection) CreateReader(config kafka.ReaderConfig) *kafka.Reader { return kafka.NewReader(config) } -func (c *Connection) CreateWriter(topic string) *kafka.Writer { +func (c *Connection) CreateWriter( + topic string, + opts ProducerOptionsConfig, + async bool, + balancer kafka.Balancer, +) *kafka.Writer { return &kafka.Writer{ Addr: kafka.TCP(c.brokers...), Topic: topic, - RequiredAcks: kafka.RequireAll, - Balancer: &kafka.LeastBytes{}, - Async: false, + RequiredAcks: kafka.RequiredAcks(opts.RequiredAcks), + Async: async, + Balancer: balancer, + BatchSize: opts.BatchSize, + BatchTimeout: opts.BatchTimeout, + WriteTimeout: opts.WriteTimeout, + ReadTimeout: opts.ReadTimeout, } } diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index b8bafd3..9ff7bf1 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -65,29 +65,24 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap readerConfig := kafka.ReaderConfig{ GroupID: c.cfg.Options.Group, Topic: topic, - CommitInterval: c.cfg.Options.Commit.Interval, + CommitInterval: c.cfg.Options.Consumer.Commit.Interval, MinBytes: minBytes, MaxBytes: maxBytes, ReadLagInterval: readLagInterval, - SessionTimeout: sessionTimeout, + SessionTimeout: c.cfg.Options.Consumer.SessionTimeout, RebalanceTimeout: rebalanceTimeout, - HeartbeatInterval: heartbeatInterval, + HeartbeatInterval: c.cfg.Options.Consumer.HeartbeatInterval, MaxWait: time.Second, } - switch c.cfg.Options.Rebalance.Strategy { + switch c.cfg.Options.Consumer.Rebalance.Strategy { case "range": - readerConfig.GroupBalancers = []kafka.GroupBalancer{ - kafka.RangeGroupBalancer{}, - } + readerConfig.GroupBalancers = []kafka.GroupBalancer{kafka.RangeGroupBalancer{}} case "roundrobin": - readerConfig.GroupBalancers = []kafka.GroupBalancer{ - kafka.RoundRobinGroupBalancer{}, - } + readerConfig.GroupBalancers = []kafka.GroupBalancer{kafka.RoundRobinGroupBalancer{}} } c.configureOffset(&readerConfig) - reader := c.conn.CreateReader(readerConfig) c.readers = append(c.readers, reader) } @@ -103,7 +98,7 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap }(reader) } - if c.cfg.Options.Commit.Strategy == "batch" || c.cfg.Options.Commit.Strategy == "deferred" { + if c.cfg.Options.Consumer.Commit.Strategy == "batch" { go c.startBatchCommitter(ctx) } @@ -122,11 +117,11 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap } func (c *Consumer) configureOffset(config *kafka.ReaderConfig) { - switch c.cfg.Options.OffsetConfig.Type { + switch c.cfg.Options.Consumer.OffsetConfig.Type { case "earliest": config.StartOffset = kafka.FirstOffset case "specific": - config.StartOffset = c.cfg.Options.OffsetConfig.Value + config.StartOffset = c.cfg.Options.Consumer.OffsetConfig.Value default: config.StartOffset = kafka.LastOffset } @@ -137,7 +132,7 @@ func (c *Consumer) startWorkerPool( jobs chan job, handler func(context.Context, api.Envelope) error, ) { - poolSize := c.cfg.Options.Pool.Size + poolSize := c.cfg.Options.Consumer.Pool.Size if poolSize <= 0 { poolSize = defaultPoolSize } @@ -147,7 +142,7 @@ func (c *Consumer) startWorkerPool( go c.startWorker(ctx, jobs, handler) } - if c.cfg.Options.Pool.Dynamic { + if c.cfg.Options.Consumer.Pool.Dynamic { go c.manageWorkerPool(ctx, jobs, handler) } } @@ -160,9 +155,9 @@ func (c *Consumer) manageWorkerPool( ticker := time.NewTicker(workerPoolCheckInterval) defer ticker.Stop() - currentSize := c.cfg.Options.Pool.Size - minSize := c.cfg.Options.Pool.MinSize - maxSize := c.cfg.Options.Pool.MaxSize + currentSize := c.cfg.Options.Consumer.Pool.Size + minSize := c.cfg.Options.Consumer.Pool.MinSize + maxSize := c.cfg.Options.Consumer.Pool.MaxSize for { select { @@ -258,7 +253,7 @@ func (c *Consumer) handleMessage( } func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka.Message) { - switch c.cfg.Options.Commit.Strategy { + switch c.cfg.Options.Consumer.Commit.Strategy { case "auto": if err := r.CommitMessages(ctx, msg); err != nil { c.logger.ErrorContext(ctx, "Failed to commit message", @@ -275,9 +270,9 @@ func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka msgKey := fmt.Sprintf("%s-%d-%d", msg.Topic, msg.Partition, msg.Offset) c.deferredCommits.Store(msgKey, messageWithReader{message: msg, reader: r}) - if len(c.batchMessages) >= c.cfg.Options.Commit.BatchSize { + if len(c.batchMessages) >= c.cfg.Options.Consumer.Commit.BatchSize { c.batchMutex.Unlock() - c.commitBatch() + c.commitBatch(ctx) } else { c.batchMutex.Unlock() } @@ -288,7 +283,7 @@ func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka } func (c *Consumer) startBatchCommitter(ctx context.Context) { - ticker := time.NewTicker(c.cfg.Options.Commit.Interval) + ticker := time.NewTicker(c.cfg.Options.Consumer.Commit.Interval) defer ticker.Stop() for { @@ -296,12 +291,12 @@ func (c *Consumer) startBatchCommitter(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - c.commitBatch() + c.commitBatch(ctx) } } } -func (c *Consumer) commitBatch() { +func (c *Consumer) commitBatch(ctx context.Context) { c.batchMutex.Lock() defer c.batchMutex.Unlock() @@ -314,7 +309,7 @@ func (c *Consumer) commitBatch() { partitionOffsets := c.findMaxOffsetPerPartition(messages) - c.commitMessagesAndCleanup(reader, messages, partitionOffsets) + c.commitMessagesAndCleanup(ctx, reader, messages, partitionOffsets) } c.batchMessages = c.batchMessages[:0] @@ -348,13 +343,14 @@ func (c *Consumer) findMaxOffsetPerPartition(messages []kafka.Message) map[int]k } func (c *Consumer) commitMessagesAndCleanup( + ctx context.Context, reader *kafka.Reader, messages []kafka.Message, partitionOffsets map[int]kafka.Message, ) { for _, msg := range partitionOffsets { - if err := reader.CommitMessages(context.Background(), msg); err != nil { - c.logger.Error("Failed to commit message batch", + if err := reader.CommitMessages(ctx, msg); err != nil { + c.logger.ErrorContext(ctx, "Failed to commit message batch", "topic", msg.Topic, "partition", msg.Partition, "offset", msg.Offset, diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index 3079aca..ad15df8 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "sync" "time" "github.com/segmentio/kafka-go" @@ -18,15 +19,67 @@ type Producer struct { serializer api.Serializer conn *Connection logger *slog.Logger + writers map[string]*kafka.Writer + mu sync.RWMutex } func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) (*Producer, error) { - return &Producer{ + p := &Producer{ cfg: cfg, serializer: ser, conn: conn, logger: logger, - }, nil + writers: make(map[string]*kafka.Writer), + } + + if len(cfg.Options.Topics) == 0 { + return nil, errors.New("no topics configured for kafka transport") + } + + var balancer kafka.Balancer = &kafka.LeastBytes{} + switch cfg.Options.Producer.Balancer { + case "hash": + balancer = &kafka.Hash{} + case "round_robin": + balancer = &kafka.RoundRobin{} + case "least_bytes": + balancer = &kafka.LeastBytes{} + } + + if cfg.Options.Key.Strategy != "none" { + balancer = &kafka.Hash{} + } + + for _, topic := range cfg.Options.Topics { + writer := conn.CreateWriter( + topic, + cfg.Options.Producer, + cfg.Options.Producer.Async, + balancer, + ) + + p.writers[topic] = writer + } + + return p, nil +} + +func (p *Producer) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + + var errs []error + for topic, writer := range p.writers { + if err := writer.Close(); err != nil { + errs = append(errs, fmt.Errorf("failed to close writer for topic %s: %w", topic, err)) + } + } + + if len(errs) > 0 { + return fmt.Errorf("errors closing writers: %v", errs) + } + + return nil } func (p *Producer) Send(ctx context.Context, env api.Envelope) error { @@ -51,49 +104,37 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { msg.Key = key } - topics := p.cfg.Options.Topics - - if len(topics) == 0 { - return errors.New("no topics configured for kafka transport") - } - - for _, topic := range topics { - writer := p.conn.CreateWriter(topic) + p.mu.RLock() + defer p.mu.RUnlock() - if p.cfg.Options.Key.Strategy != "none" { - writer.Balancer = &kafka.Hash{} + for _, topic := range p.cfg.Options.Topics { + writer, exists := p.writers[topic] + if !exists { + return fmt.Errorf("writer for topic %s not found", topic) } if writeErr := writer.WriteMessages(ctx, msg); writeErr != nil { - return fmt.Errorf("producer failed to write messages: %w", writeErr) + return fmt.Errorf("producer failed to write messages to topic %s: %w", topic, writeErr) } p.logger.DebugContext(ctx, "message sent to kafka topic", slog.String("topic", topic), slog.String("message_type", fmt.Sprintf("%T", env.Message()))) - - closeErr := writer.Close() - if closeErr != nil { - return closeErr - } } return nil } func (p *Producer) extractMessageKey(env api.Envelope) ([]byte, error) { - switch p.cfg.Options.Key.Strategy { - case "none": + if p.cfg.Options.Key.Strategy != "message_id" { return nil, nil - case "message_id": - for _, s := range env.Stamps() { - if msgIDStamp, ok := s.(stamps.MessageIDStamp); ok { - return []byte(msgIDStamp.MessageID), nil - } - } + } - return nil, errors.New("message_id stamp not found") - default: - return nil, fmt.Errorf("unknown key strategy: %s", p.cfg.Options.Key.Strategy) + for _, s := range env.Stamps() { + if msgIDStamp, ok := s.(stamps.MessageIDStamp); ok { + return []byte(msgIDStamp.MessageID), nil + } } + + return nil, errors.New("message_id stamp not found") } diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index 3aa2c5f..c27433d 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -63,6 +63,10 @@ func (t *Transport) Receive(ctx context.Context, handler func(context.Context, a return t.consumer.Consume(ctx, handler) } +func (t *Transport) Close() error { + return t.producer.Close() +} + func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { return t.producer.Send(ctx, env) } diff --git a/transport/redis/benchmark_test.go b/transport/redis/benchmark_test.go new file mode 100644 index 0000000..8c19d83 --- /dev/null +++ b/transport/redis/benchmark_test.go @@ -0,0 +1,172 @@ +package redis_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/config" +) + +const ( + benchmarkDSN = "redis://localhost:6379/0" + benchmarkTimeout = 60 * time.Second +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func (m *BenchmarkMessage) RoutingKey() string { + return "benchmark_routing_key" +} + +type BenchmarkHandler struct { + processedCount *int64 + wg *sync.WaitGroup +} + +func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { + atomic.AddInt64(h.processedCount, 1) + if h.wg != nil { + h.wg.Done() + } + + return nil +} + +func (h *BenchmarkHandler) GetBusName() string { + return "default" +} + +func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { + b.Helper() + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "redis": { + DSN: benchmarkDSN, + Serializer: "default.transport.serializer", + Options: map[string]any{ + "auto_setup": true, + "stream": "benchmark_stream", + "group": "benchmark_group", + "consumer": "benchmark_consumer", + }, + }, + }, + Routing: map[string]string{ + "*redis_test.BenchmarkMessage": "redis", + }, + } + + processedCount := int64(0) + var wg *sync.WaitGroup + if withWaitGroup { + wg = &sync.WaitGroup{} + } + + handler := &BenchmarkHandler{ + processedCount: &processedCount, + wg: wg, + } + + builderInstance := builder.NewBuilder(cfg, logger) + if err := builderInstance.RegisterHandler(handler); err != nil { + b.Fatalf("Register handler failed: %v", err) + } + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) + go func() { + defer cancel() + if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { + b.Logf("Messenger run error: %v", runErr) + } + }() + + time.Sleep(2 * time.Second) + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, size, false) + }) + } +} diff --git a/transport/sync/benchmark_test.go b/transport/sync/benchmark_test.go new file mode 100644 index 0000000..9eb1dd8 --- /dev/null +++ b/transport/sync/benchmark_test.go @@ -0,0 +1,166 @@ +package sync_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/config" +) + +const ( + benchmarkDSN = "sync://" + benchmarkTimeout = 60 * time.Second +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func (m *BenchmarkMessage) RoutingKey() string { + return "benchmark_routing_key" +} + +type BenchmarkHandler struct { + processedCount *int64 + wg *sync.WaitGroup +} + +func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { + atomic.AddInt64(h.processedCount, 1) + if h.wg != nil { + h.wg.Done() + } + + return nil +} + +func (h *BenchmarkHandler) GetBusName() string { + return "default" +} + +func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { + b.Helper() + + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "sync": { + DSN: benchmarkDSN, + Serializer: "default.transport.serializer", + }, + }, + Routing: map[string]string{ + "*sync_test.BenchmarkMessage": "sync", + }, + } + + processedCount := int64(0) + var wg *sync.WaitGroup + if withWaitGroup { + wg = &sync.WaitGroup{} + } + + handler := &BenchmarkHandler{ + processedCount: &processedCount, + wg: wg, + } + + builderInstance := builder.NewBuilder(cfg, logger) + if err := builderInstance.RegisterHandler(handler); err != nil { + b.Fatalf("Register handler failed: %v", err) + } + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) + go func() { + defer cancel() + if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { + b.Logf("Messenger run error: %v", runErr) + } + }() + + time.Sleep(2 * time.Second) + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupMessenger(b, false) + dispatchMessages(b, bus, size, false) + }) + } +} From c2730b0d97aa93f41eb1cef6b9b12485d36933f3 Mon Sep 17 00:00:00 2001 From: Gerfey Date: Sat, 9 Aug 2025 14:47:52 +0700 Subject: [PATCH 15/17] refactor: migrating code to core/builder and adding benchmarks --- README.md | 7 +- README.ru.md | 7 +- {builder => core/builder}/builder.go | 12 +-- {builder => core/builder}/builder_test.go | 5 +- {builder => core/builder}/resolver.go | 0 {builder => core/builder}/resolver_test.go | 3 +- {config => core/config}/config.go | 0 {config => core/config}/config_test.go | 10 +-- {config => core/config}/env_processor.go | 0 {config => core/config}/env_processor_test.go | 2 +- {config => core/config}/parser.go | 0 {config => core/config}/parser_test.go | 4 +- {config => core/config}/reader.go | 0 {config => core/config}/reader_test.go | 16 ++-- core/serializer/serializer.go | 2 +- core/serializer/serializer_test.go | 3 +- docs/benchmark/AMQP-Benchmark.md | 47 ++++------- docs/benchmark/Kafka-async-Benchmark.md | 28 +++++++ docs/benchmark/Redis-Benchmark.md | 24 ++++++ docs/benchmark/Sync-Benchmark.md | 24 ++++++ examples/config/config.go | 3 +- examples/kafka_transport/kafka_transport.go | 4 +- examples/kafka_transport/messenger.yaml | 12 +-- examples/messenger/messenger.go | 4 +- examples/messenger/messenger.yaml | 3 +- examples/redis_transport/messenger.yaml | 2 +- examples/redis_transport/redis_transport.go | 4 +- examples/retry_messenger/messenger.yaml | 2 +- examples/retry_messenger/retry_messenger.go | 4 +- examples/serializer/handler/hello_handler.go | 20 ----- examples/serializer/message/hello.go | 9 --- examples/serializer/messenger.yaml | 10 --- examples/serializer/serializer.go | 69 ---------------- examples/serializer/serializer/serializer.go | 38 --------- examples/sync_transport/sync_transport.go | 4 +- tests/e2e/happy_path_test.go | 5 +- tests/e2e/simple_test.go | 5 +- tests/fixtures/configs/config_with_env.yaml | 2 +- tests/fixtures/configs/e2e.yaml | 2 +- .../fixtures/configs/multiple_transports.yaml | 2 +- tests/fixtures/configs/valid_config.yaml | 4 +- transport/amqp/benchmark_test.go | 68 +++------------- transport/amqp/config.go | 7 +- transport/amqp/consumer.go | 39 ---------- transport/chain.go | 3 +- transport/chain_test.go | 5 +- transport/kafka/benchmark_test.go | 78 ++++--------------- transport/kafka/config.go | 22 +++--- transport/kafka/connection.go | 19 ++--- transport/kafka/consumer.go | 39 ---------- transport/kafka/producer.go | 4 - transport/redis/benchmark_test.go | 72 +++-------------- transport/sync/benchmark_test.go | 72 +++-------------- 53 files changed, 234 insertions(+), 596 deletions(-) rename {builder => core/builder}/builder.go (98%) rename {builder => core/builder}/builder_test.go (99%) rename {builder => core/builder}/resolver.go (100%) rename {builder => core/builder}/resolver_test.go (99%) rename {config => core/config}/config.go (100%) rename {config => core/config}/config_test.go (94%) rename {config => core/config}/env_processor.go (100%) rename {config => core/config}/env_processor_test.go (99%) rename {config => core/config}/parser.go (100%) rename {config => core/config}/parser_test.go (97%) rename {config => core/config}/reader.go (100%) rename {config => core/config}/reader_test.go (89%) create mode 100644 docs/benchmark/Kafka-async-Benchmark.md create mode 100644 docs/benchmark/Redis-Benchmark.md create mode 100644 docs/benchmark/Sync-Benchmark.md delete mode 100644 examples/serializer/handler/hello_handler.go delete mode 100644 examples/serializer/message/hello.go delete mode 100644 examples/serializer/messenger.yaml delete mode 100644 examples/serializer/serializer.go delete mode 100644 examples/serializer/serializer/serializer.go diff --git a/README.md b/README.md index 55501e3..d25a85b 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,12 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) > See [Usage Scenarios](https://github.com/Gerfey/messenger/wiki/Usage-Scenarios) for commands, queries, return values and advanced use-cases. -## Efficiency +## Benchmark -- AMQP (RabbitMQ): [AMQP Benchmark Report](docs/benchmark/AMQP-Benchmark.md) +- AMQP (RabbitMQ): [AMQP Transport Benchmark Report](docs/benchmark/AMQP-Benchmark.md) +- Redis (Stream): [Redis Transport Benchmark Report](docs/benchmark/Redis-Benchmark.md) +- Sync: [Sync Transport Benchmark Report](docs/benchmark/Sync-Benchmark.md) +- Kafka (Async): [Kafka Transport Async Benchmark Report](docs/benchmark/Kafka-async-Benchmark.md) ## Contributing diff --git a/README.ru.md b/README.ru.md index 9b85781..859f0f7 100644 --- a/README.ru.md +++ b/README.ru.md @@ -95,9 +95,12 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) > Смотри [Сценарии использования](https://github.com/Gerfey/messenger/wiki/Сценарии-использования). -## Производительность +## Показатели -- AMQP (RabbitMQ): [AMQP Benchmark Report](docs/benchmark/AMQP-Benchmark.md) +- AMQP (RabbitMQ): [AMQP Transport Benchmark Report](docs/benchmark/AMQP-Benchmark.md) +- Redis (Stream): [Redis Transport Benchmark Report](docs/benchmark/Redis-Benchmark.md) +- Sync: [Sync Transport Benchmark Report](docs/benchmark/Sync-Benchmark.md) +- Kafka (Async): [Kafka Transport Async Benchmark Report](docs/benchmark/Kafka-async-Benchmark.md) ## Как внести вклад diff --git a/builder/builder.go b/core/builder/builder.go similarity index 98% rename from builder/builder.go rename to core/builder/builder.go index 672e72d..e36b7a8 100644 --- a/builder/builder.go +++ b/core/builder/builder.go @@ -8,8 +8,8 @@ import ( "github.com/gerfey/messenger" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/core/bus" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/core/event" "github.com/gerfey/messenger/core/handler" "github.com/gerfey/messenger/core/listener" @@ -77,10 +77,6 @@ func (b *Builder) RegisterHandler(handler any) error { return fmt.Errorf("register handler: %w", err) } - for _, h := range b.handlersLocator.GetAll() { - b.resolver.Register(h.InputType.String(), h.InputType) - } - return nil } @@ -107,6 +103,10 @@ func (b *Builder) RegisterListener(event any, listener any) { } func (b *Builder) Build() (api.Messenger, error) { + for _, h := range b.handlersLocator.GetAll() { + b.resolver.Register(h.InputType.String(), h.InputType) + } + b.registerStamps() b.serializerLocator.Register("default.transport.serializer", serializer.NewSerializer(b.resolver)) @@ -180,7 +180,7 @@ func (b *Builder) setupRouting() (api.Router, error) { router := routing.NewRouter() for msgTypeStr, transportName := range b.cfg.Routing { - t, err := b.handlersLocator.ResolveMessageType(msgTypeStr) + t, err := b.resolver.ResolveMessageType(msgTypeStr) if err != nil { return nil, fmt.Errorf("failed to resolve message type '%s' in routing configuration: %w", msgTypeStr, err) } diff --git a/builder/builder_test.go b/core/builder/builder_test.go similarity index 99% rename from builder/builder_test.go rename to core/builder/builder_test.go index c6e9956..23d0ea3 100644 --- a/builder/builder_test.go +++ b/core/builder/builder_test.go @@ -7,8 +7,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/core/stamps" "github.com/gerfey/messenger/tests/helpers" ) diff --git a/builder/resolver.go b/core/builder/resolver.go similarity index 100% rename from builder/resolver.go rename to core/builder/resolver.go diff --git a/builder/resolver_test.go b/core/builder/resolver_test.go similarity index 99% rename from builder/resolver_test.go rename to core/builder/resolver_test.go index ea6686a..e7e7b42 100644 --- a/builder/resolver_test.go +++ b/core/builder/resolver_test.go @@ -7,7 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/stamps" "github.com/gerfey/messenger/tests/helpers" ) diff --git a/config/config.go b/core/config/config.go similarity index 100% rename from config/config.go rename to core/config/config.go diff --git a/config/config_test.go b/core/config/config_test.go similarity index 94% rename from config/config_test.go rename to core/config/config_test.go index ffb0c69..3d1c436 100644 --- a/config/config_test.go +++ b/core/config/config_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) type testProcessor struct { @@ -25,7 +25,7 @@ func (p *testProcessor) Process(content []byte) ([]byte, error) { func TestLoadConfig(t *testing.T) { t.Run("load valid config", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" cfg, err := config.LoadConfig(path) require.NoError(t, err) @@ -37,7 +37,7 @@ func TestLoadConfig(t *testing.T) { }) t.Run("load config with env variables", func(t *testing.T) { - path := "../tests/fixtures/configs/config_with_env.yaml" + path := "../../tests/fixtures/configs/config_with_env.yaml" t.Setenv("MEMORY_HOST", "test-host") t.Setenv("CONSUMER_POOL_SIZE", "15") @@ -68,7 +68,7 @@ func TestLoadConfig(t *testing.T) { }) t.Run("load invalid config", func(t *testing.T) { - path := "../tests/fixtures/configs/invalid_config.yaml" + path := "../../tests/fixtures/configs/invalid_config.yaml" cfg, err := config.LoadConfig(path) require.Error(t, err) @@ -76,7 +76,7 @@ func TestLoadConfig(t *testing.T) { }) t.Run("load with custom processor", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" customProcessor := &testProcessor{ processFunc: func(content []byte) ([]byte, error) { diff --git a/config/env_processor.go b/core/config/env_processor.go similarity index 100% rename from config/env_processor.go rename to core/config/env_processor.go diff --git a/config/env_processor_test.go b/core/config/env_processor_test.go similarity index 99% rename from config/env_processor_test.go rename to core/config/env_processor_test.go index b166454..4589de8 100644 --- a/config/env_processor_test.go +++ b/core/config/env_processor_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) func TestEnvVarProcessor_Process(t *testing.T) { diff --git a/config/parser.go b/core/config/parser.go similarity index 100% rename from config/parser.go rename to core/config/parser.go diff --git a/config/parser_test.go b/core/config/parser_test.go similarity index 97% rename from config/parser_test.go rename to core/config/parser_test.go index 5e53130..3a84814 100644 --- a/config/parser_test.go +++ b/core/config/parser_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) func TestYAMLParser_Parse(t *testing.T) { @@ -179,7 +179,7 @@ func TestYAMLParser_Integration(t *testing.T) { reader := &config.FileReader{} t.Run("parse real config file", func(t *testing.T) { - content, err := reader.Read("../tests/fixtures/configs/valid_config.yaml") + content, err := reader.Read("../../tests/fixtures/configs/valid_config.yaml") require.NoError(t, err) var cfg config.MessengerConfig diff --git a/config/reader.go b/core/config/reader.go similarity index 100% rename from config/reader.go rename to core/config/reader.go diff --git a/config/reader_test.go b/core/config/reader_test.go similarity index 89% rename from config/reader_test.go rename to core/config/reader_test.go index 5dff18a..454306a 100644 --- a/config/reader_test.go +++ b/core/config/reader_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) type MockProcessor struct { @@ -27,7 +27,7 @@ func TestFileReader_Read(t *testing.T) { reader := &config.FileReader{} t.Run("read existing file", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" content, err := reader.Read(path) require.NoError(t, err) @@ -45,7 +45,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with single processor", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" processor := &MockProcessor{ ProcessFunc: func(content []byte) ([]byte, error) { return append(content, []byte("\n# Processed")...), nil @@ -59,7 +59,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with multiple processors", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" processor1 := &MockProcessor{ ProcessFunc: func(content []byte) ([]byte, error) { return append(content, []byte("\n# Processor1")...), nil @@ -79,7 +79,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with processor error", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" expectedErr := errors.New("processor error") processor := &MockProcessor{ ProcessFunc: func(_ []byte) ([]byte, error) { @@ -95,7 +95,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with multiple processors, second fails", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" expectedErr := errors.New("processor 2 error") processor1 := &MockProcessor{ ProcessFunc: func(content []byte) ([]byte, error) { @@ -135,7 +135,7 @@ func TestFileReader_Integration(t *testing.T) { reader := &config.FileReader{} t.Run("read config with env processor", func(t *testing.T) { - path := "../tests/fixtures/configs/config_with_env.yaml" + path := "../../tests/fixtures/configs/config_with_env.yaml" t.Setenv("MEMORY_HOST", "localhost") t.Setenv("CONSUMER_POOL_SIZE", "20") @@ -150,7 +150,7 @@ func TestFileReader_Integration(t *testing.T) { }) t.Run("read config with multiple processors", func(t *testing.T) { - path := "../tests/fixtures/configs/config_with_env.yaml" + path := "../../tests/fixtures/configs/config_with_env.yaml" t.Setenv("MEMORY_HOST", "localhost") diff --git a/core/serializer/serializer.go b/core/serializer/serializer.go index 7ae0dae..c22501d 100644 --- a/core/serializer/serializer.go +++ b/core/serializer/serializer.go @@ -6,7 +6,7 @@ import ( "reflect" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/core/envelope" ) diff --git a/core/serializer/serializer_test.go b/core/serializer/serializer_test.go index 988dd23..2352789 100644 --- a/core/serializer/serializer_test.go +++ b/core/serializer/serializer_test.go @@ -6,9 +6,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/serializer" - "github.com/gerfey/messenger/builder" "github.com/gerfey/messenger/core/envelope" "github.com/gerfey/messenger/tests/helpers" ) diff --git a/docs/benchmark/AMQP-Benchmark.md b/docs/benchmark/AMQP-Benchmark.md index 595ea2b..c5a7056 100644 --- a/docs/benchmark/AMQP-Benchmark.md +++ b/docs/benchmark/AMQP-Benchmark.md @@ -1,45 +1,26 @@ -# AMQP Benchmark Report +# AMQP Transport Benchmark Report -Current performance results of the AMQP transport in Messenger (`v0.8.0`), tested with `RabbitMQ` using the `amqp091-go` client. +* Transport: `amqp://` (Messenger `v0.8.0`) +* Publishing mode: sync ## Overall Performance -| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | -|------------------------------|---------------|---------------------|----------------|-----------| -| `BenchmarkAMQPSend` | 465,508 | ~2,148 | 13,822 | 267 | -| `BenchmarkAMQPConcurrentSend`| 296,922 | ~3,368 | 13,829 | 266 | +| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------------------- | -----------: | -------------------: | ------------: | --------: | +| `BenchmarkSend` | 378,376 | \~2,643 | 6,792 | 130 | +| `BenchmarkConcurrentSend` | 245,631 | \~4,071 | 6,790 | 129 | -*Concurrent sending is ~36% faster with 100B messages.* +*Parallel sending gives ~1.5x increase in bandwidth with similar memory consumption.* --- ## Message Size Impact | Message Size | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | -|------------------|---------------|---------------------|----------------|-----------| -| 100 B | 472,831 | ~2,115 | 13,826 | 267 | -| 1 KB | 557,170 | ~1,794 | 19,874 | 267 | -| 10 KB | 704,831 | ~1,418 | 82,185 | 269 | -| 100 KB | 1,788,004 | ~559 | 726,393 | 276 | - -*As the payload size increases, throughput decreases and memory pressure on GC grows, as expected.* - ---- - -## Allocations: `pprof` Analysis -> Collected using `go test -bench=BenchmarkAMQP -benchmem -memprofile mem.out`, analyzed via `pprof`. - -(pprof) top - -- encoding/json: ~20% -- amqp091-go (sendOpen, Ack, readLongstr): ~25% -- Envelope.WithStamp: 5.75% -- Middleware chain: ~5% - ---- - -## Summary - -- AMQP transport in Messenger demonstrates stable and predictable performance -- Memory and allocation optimization opportunities are being addressed in upcoming versions +| ------------ | -----------: | -------------------: | ------------: | --------: | +| 100 B | 384,571 | \~2,600 | 6,797 | 130 | +| 1 KB | 397,139 | \~2,518 | 9,058 | 130 | +| 10 KB | 445,552 | \~2,244 | 31,667 | 130 | +| 100 KB | 1,397,238 | \~716 | 320,669 | 136 | +*Increasing the payload size is expected to reduce RPS and increase memory consumption.* diff --git a/docs/benchmark/Kafka-async-Benchmark.md b/docs/benchmark/Kafka-async-Benchmark.md new file mode 100644 index 0000000..273781e --- /dev/null +++ b/docs/benchmark/Kafka-async-Benchmark.md @@ -0,0 +1,28 @@ +# Kafka Transport Async Benchmark Report + +* Transport: `kafka://` (Messenger `v0.8.0`) +* Publishing mode: async (`async=true`) + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------------------- | -----------: | -------------------: | ------------: | --------: | +| `BenchmarkSend` | 3,018 | \~331,345 | 3,072 | 47 | +| `BenchmarkConcurrentSend` | 1,859 | \~537,924 | 3,067 | 47 | + + +*Async publishing provides high throughput with a stable allocation profile.* + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------ | -----------: | -------------------: | ------------: | --------: | +| 100 B | 2,913 | \~343,289 | 3,062 | 47 | +| 1 KB | 4,147 | \~241,138 | 5,304 | 47 | +| 10 KB | 9,800 | \~102,041 | 27,723 | 47 | +| 100 KB | 87,745 | \~11,397 | 254,397 | 48 | + + +*Increasing the message size is expected to reduce throughput and increase memory consumption, but the curve looks smooth and predictable.* diff --git a/docs/benchmark/Redis-Benchmark.md b/docs/benchmark/Redis-Benchmark.md new file mode 100644 index 0000000..d3d3e69 --- /dev/null +++ b/docs/benchmark/Redis-Benchmark.md @@ -0,0 +1,24 @@ +# Redis Transport Benchmark Report + +* Transport: `redis://` (Messenger `v0.8.0`) +* Publishing mode: sync + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/s) | Memory (B/op) | Allocs/op | +| ------------------------- | ------------ | ------------------ | ------------- | --------- | +| `BenchmarkSend` | 133,748 | \~7,477 | 7,361 | 106 | +| `BenchmarkConcurrentSend` | 32,616 | \~30,675 | 7,397 | 106 | + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/s) | Memory (B/op) | Allocs/op | +| ------------ | ------------ | ------------------ | ------------- | --------- | +| 100 B | 127,118 | \~7,866 | 7,362 | 106 | +| 1 KB | 139,020 | \~7,193 | 9,619 | 106 | +| 10 KB | 219,779 | \~4,551 | 32,072 | 106 | +| 100 KB | 1,009,267 | \~991 | 274,182 | 107 | + +*Redis Streams shows good rps in parallel, but as the payload increases, the speed decreases and memory increases.* diff --git a/docs/benchmark/Sync-Benchmark.md b/docs/benchmark/Sync-Benchmark.md new file mode 100644 index 0000000..0675440 --- /dev/null +++ b/docs/benchmark/Sync-Benchmark.md @@ -0,0 +1,24 @@ +# Sync Transport Benchmark Report + +* Transport: `sync://` (Messenger `v0.8.0`) +* Publishing mode: sync + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------------------- | -----------: | -------------------: | ------------: | --------: | +| `BenchmarkSend` | 1,110 | \~900,901 | 1,368 | 38 | +| `BenchmarkConcurrentSend` | 932.9 | \~1,071,926 | 1,377 | 38 | + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------ | -----------: | -------------------: | ------------: | --------: | +| 100 B | 1,152 | \~868,056 | 1,368 | 38 | +| 1 KB | 1,250 | \~800,000 | 2,281 | 38 | +| 10 KB | 2,091 | \~478,240 | 11,505 | 38 | +| 100 KB | 11,583 | \~86,333 | 107,832 | 38 | + +*ultra‑low latency and allocations — ideal for unit tests, CQRS commands/queries and inline middleware.* diff --git a/examples/config/config.go b/examples/config/config.go index 8cbc630..f402025 100644 --- a/examples/config/config.go +++ b/examples/config/config.go @@ -8,7 +8,8 @@ import ( "gopkg.in/yaml.v3" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/transport/amqp" ) diff --git a/examples/kafka_transport/kafka_transport.go b/examples/kafka_transport/kafka_transport.go index d7aedc8..95ab734 100644 --- a/examples/kafka_transport/kafka_transport.go +++ b/examples/kafka_transport/kafka_transport.go @@ -5,8 +5,8 @@ import ( "log/slog" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/examples/kafka_transport/handler" "github.com/gerfey/messenger/examples/kafka_transport/message" ) diff --git a/examples/kafka_transport/messenger.yaml b/examples/kafka_transport/messenger.yaml index 892b87e..94d5bec 100644 --- a/examples/kafka_transport/messenger.yaml +++ b/examples/kafka_transport/messenger.yaml @@ -15,12 +15,13 @@ transports: producer: async: true - required_acks: -1 # -1: all replicas (strongest durability), 0: no ack, 1: leader only - batch_size: 1000 - batch_timeout: 20ms + auto_topic_creation: true + required_acks: 1 # -1: all replicas (strongest durability), 0: no ack, 1: leader only + batch_size: 256 + batch_timeout: 10ms write_timeout: 10s read_timeout: 10s - balancer: least_bytes # least_bytes, hash, round_robin + balancer: round_robin # least_bytes, hash, round_robin consumer: offset: @@ -36,7 +37,6 @@ transports: size: 3 min_size: 2 max_size: 10 - dynamic: true session_timeout: 10s heartbeat_interval: 2s @@ -44,4 +44,4 @@ transports: strategy: message_id # none / message_id routing: - message.ExampleHelloMessage: kafka + "*message.ExampleHelloMessage": kafka diff --git a/examples/messenger/messenger.go b/examples/messenger/messenger.go index c0a25c0..3dd6978 100644 --- a/examples/messenger/messenger.go +++ b/examples/messenger/messenger.go @@ -5,8 +5,8 @@ import ( "log/slog" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/examples/messenger/handler" "github.com/gerfey/messenger/examples/messenger/message" "github.com/gerfey/messenger/examples/messenger/middleware" diff --git a/examples/messenger/messenger.yaml b/examples/messenger/messenger.yaml index 72d1501..79ae45a 100644 --- a/examples/messenger/messenger.yaml +++ b/examples/messenger/messenger.yaml @@ -19,7 +19,6 @@ transports: size: 10 min_size: 5 max_size: 20 - dynamic: true exchange: name: test.exchange type: topic @@ -29,4 +28,4 @@ transports: - test_routing_key routing: - message.ExampleHelloMessage: amqp + "*message.ExampleHelloMessage": amqp diff --git a/examples/redis_transport/messenger.yaml b/examples/redis_transport/messenger.yaml index a9b2867..a883c19 100644 --- a/examples/redis_transport/messenger.yaml +++ b/examples/redis_transport/messenger.yaml @@ -29,4 +29,4 @@ transports: failed_messages_queue: ~ routing: - message.ExampleHelloMessage: redis + "*message.ExampleHelloMessage": redis diff --git a/examples/redis_transport/redis_transport.go b/examples/redis_transport/redis_transport.go index ed9c097..0791c5f 100644 --- a/examples/redis_transport/redis_transport.go +++ b/examples/redis_transport/redis_transport.go @@ -5,8 +5,8 @@ import ( "log/slog" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/examples/redis_transport/handler" "github.com/gerfey/messenger/examples/redis_transport/message" ) diff --git a/examples/retry_messenger/messenger.yaml b/examples/retry_messenger/messenger.yaml index 079887b..9fe82db 100644 --- a/examples/retry_messenger/messenger.yaml +++ b/examples/retry_messenger/messenger.yaml @@ -34,4 +34,4 @@ transports: failed_messages_queue: ~ routing: - message.ExampleHelloMessage: amqp + "*message.ExampleHelloMessage": amqp diff --git a/examples/retry_messenger/retry_messenger.go b/examples/retry_messenger/retry_messenger.go index 162cf2a..9cc3caf 100644 --- a/examples/retry_messenger/retry_messenger.go +++ b/examples/retry_messenger/retry_messenger.go @@ -5,8 +5,8 @@ import ( "log/slog" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/core/event" "github.com/gerfey/messenger/examples/retry_messenger/handler" "github.com/gerfey/messenger/examples/retry_messenger/listener" diff --git a/examples/serializer/handler/hello_handler.go b/examples/serializer/handler/hello_handler.go deleted file mode 100644 index b2d42f6..0000000 --- a/examples/serializer/handler/hello_handler.go +++ /dev/null @@ -1,20 +0,0 @@ -package handler - -import ( - "context" - "fmt" - - "github.com/gerfey/messenger/examples/serializer/message" -) - -type ExampleHelloHandler struct{} - -func (u *ExampleHelloHandler) Handle(_ context.Context, msg *message.ExampleHelloMessage) error { - fmt.Printf("Handled: Text=%v\n", msg.Text) - - return nil -} - -func (u *ExampleHelloHandler) GetBusName() string { - return "default" -} diff --git a/examples/serializer/message/hello.go b/examples/serializer/message/hello.go deleted file mode 100644 index a7877ee..0000000 --- a/examples/serializer/message/hello.go +++ /dev/null @@ -1,9 +0,0 @@ -package message - -type ExampleHelloMessage struct { - Text string -} - -func (m *ExampleHelloMessage) RoutingKey() string { - return "test_routing_key" -} diff --git a/examples/serializer/messenger.yaml b/examples/serializer/messenger.yaml deleted file mode 100644 index 953745d..0000000 --- a/examples/serializer/messenger.yaml +++ /dev/null @@ -1,10 +0,0 @@ -default_bus: default -default_serializer: default.transport.serializer - -buses: - default: ~ - -transports: - sync: - dsn: "sync://" - serializer: test.json diff --git a/examples/serializer/serializer.go b/examples/serializer/serializer.go deleted file mode 100644 index 568502d..0000000 --- a/examples/serializer/serializer.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "context" - "log/slog" - "time" - - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" - "github.com/gerfey/messenger/examples/serializer/handler" - "github.com/gerfey/messenger/examples/serializer/message" - "github.com/gerfey/messenger/examples/serializer/serializer" -) - -const ( - waitDurationSeconds = 20 -) - -func main() { - ctx := context.Background() - - log := slog.Default() - - cfg, err := config.LoadConfig("./examples/serializer/messenger.yaml") - if err != nil { - log.Error("ERROR load config", "error", err) - - return - } - - b := builder.NewBuilder(cfg, log) - - _ = b.RegisterHandler(&handler.ExampleHelloHandler{}) - - b.RegisterSerializer("test.json", serializer.NewTestSerializer()) - - messenger, err := b.Build() - if err != nil { - log.Error("failed to build messenger", "error", err) - - return - } - - go func() { - if runErr := messenger.Run(ctx); runErr != nil { - log.Error("messenger run failed", "error", runErr) - - return - } - }() - - messengerBus, err := messenger.GetDefaultBus() - if err != nil { - log.Error("failed to get default bus", "error", err) - - return - } - - _, err = messengerBus.Dispatch(ctx, &message.ExampleHelloMessage{ - Text: "Hello World", - }) - if err != nil { - log.Error("failed to dispatch message", "error", err) - - return - } - - time.Sleep(waitDurationSeconds * time.Second) -} diff --git a/examples/serializer/serializer/serializer.go b/examples/serializer/serializer/serializer.go deleted file mode 100644 index 2654808..0000000 --- a/examples/serializer/serializer/serializer.go +++ /dev/null @@ -1,38 +0,0 @@ -package serializer - -import ( - "encoding/json" - "fmt" - "reflect" - - "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/core/envelope" -) - -type TestSerializer struct{} - -func NewTestSerializer() api.Serializer { - return &TestSerializer{} -} - -func (s *TestSerializer) Marshal(env api.Envelope) ([]byte, map[string]string, error) { - msg := env.Message() - body, err := json.Marshal(msg) - if err != nil { - return nil, nil, err - } - - headers := map[string]string{ - "type": reflect.TypeOf(msg).String(), - } - - return body, headers, nil -} - -func (s *TestSerializer) Unmarshal(_ []byte, _ map[string]string) (api.Envelope, error) { - fmt.Println("TestJsonSerializer.Unmarshal") - - env := envelope.NewEnvelope("") - - return env, nil -} diff --git a/examples/sync_transport/sync_transport.go b/examples/sync_transport/sync_transport.go index 5bf59f2..b4afdc7 100644 --- a/examples/sync_transport/sync_transport.go +++ b/examples/sync_transport/sync_transport.go @@ -5,8 +5,8 @@ import ( "log/slog" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/examples/sync_transport/handler" "github.com/gerfey/messenger/examples/sync_transport/message" ) diff --git a/tests/e2e/happy_path_test.go b/tests/e2e/happy_path_test.go index 5d2f419..9609a39 100644 --- a/tests/e2e/happy_path_test.go +++ b/tests/e2e/happy_path_test.go @@ -10,10 +10,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/tests/fixtures/handlers" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" testHelpers "github.com/gerfey/messenger/tests/helpers" ) diff --git a/tests/e2e/simple_test.go b/tests/e2e/simple_test.go index 8b415c7..c2c4e32 100644 --- a/tests/e2e/simple_test.go +++ b/tests/e2e/simple_test.go @@ -6,10 +6,11 @@ import ( "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/tests/fixtures/handlers" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" testHelpers "github.com/gerfey/messenger/tests/helpers" ) diff --git a/tests/fixtures/configs/config_with_env.yaml b/tests/fixtures/configs/config_with_env.yaml index 169731d..524421f 100644 --- a/tests/fixtures/configs/config_with_env.yaml +++ b/tests/fixtures/configs/config_with_env.yaml @@ -31,4 +31,4 @@ transports: - "#" routing: - "test.TestMessage": default + "*test.TestMessage": default diff --git a/tests/fixtures/configs/e2e.yaml b/tests/fixtures/configs/e2e.yaml index 9e1d75b..b7ce93b 100644 --- a/tests/fixtures/configs/e2e.yaml +++ b/tests/fixtures/configs/e2e.yaml @@ -10,4 +10,4 @@ transports: dsn: "in-memory://" routing: - helpers.TestMessage: in-memory + "*helpers.TestMessage": in-memory diff --git a/tests/fixtures/configs/multiple_transports.yaml b/tests/fixtures/configs/multiple_transports.yaml index 02e2ea7..e966bbc 100644 --- a/tests/fixtures/configs/multiple_transports.yaml +++ b/tests/fixtures/configs/multiple_transports.yaml @@ -12,4 +12,4 @@ transports: dsn: "in-memory://" routing: - helpers.TestMessage: in-memory1 + "*helpers.TestMessage": in-memory1 diff --git a/tests/fixtures/configs/valid_config.yaml b/tests/fixtures/configs/valid_config.yaml index bde7ab1..d46bc7c 100644 --- a/tests/fixtures/configs/valid_config.yaml +++ b/tests/fixtures/configs/valid_config.yaml @@ -25,5 +25,5 @@ transports: consumer_pool_size: 5 routing: - "test.TestMessage": default - "test.AsyncMessage": async + "*test.TestMessage": default + "*test.AsyncMessage": async diff --git a/transport/amqp/benchmark_test.go b/transport/amqp/benchmark_test.go index 5d3197f..5937161 100644 --- a/transport/amqp/benchmark_test.go +++ b/transport/amqp/benchmark_test.go @@ -1,24 +1,18 @@ package amqp_test import ( - "context" - "errors" "fmt" "log/slog" - "os" "sync" - "sync/atomic" "testing" - "time" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" ) const ( - benchmarkDSN = "amqp://guest:guest@localhost:5672/" - benchmarkTimeout = 60 * time.Second + benchmarkDSN = "amqp://guest:guest@localhost:5672/" ) type BenchmarkMessage struct { @@ -31,30 +25,10 @@ func (m *BenchmarkMessage) RoutingKey() string { return "benchmark_routing_key" } -type BenchmarkHandler struct { - processedCount *int64 - wg *sync.WaitGroup -} - -func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { - atomic.AddInt64(h.processedCount, 1) - if h.wg != nil { - h.wg.Done() - } - - return nil -} - -func (h *BenchmarkHandler) GetBusName() string { - return "default" -} - -func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { +func setupMessenger(b *testing.B) api.MessageBus { b.Helper() - logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ - Level: slog.LevelError, - })) + logger := slog.New(slog.DiscardHandler) cfg := &config.MessengerConfig{ DefaultBus: "default", @@ -84,37 +58,15 @@ func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { }, } - processedCount := int64(0) - var wg *sync.WaitGroup - if withWaitGroup { - wg = &sync.WaitGroup{} - } - - handler := &BenchmarkHandler{ - processedCount: &processedCount, - wg: wg, - } - builderInstance := builder.NewBuilder(cfg, logger) - if err := builderInstance.RegisterHandler(handler); err != nil { - b.Fatalf("Register handler failed: %v", err) - } + + builderInstance.RegisterMessage(&BenchmarkMessage{}) messenger, err := builderInstance.Build() if err != nil { b.Fatalf("Build messenger failed: %v", err) } - ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) - go func() { - defer cancel() - if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { - b.Logf("Messenger run error: %v", runErr) - } - }() - - time.Sleep(2 * time.Second) - bus, err := messenger.GetDefaultBus() if err != nil { b.Fatalf("Get default bus failed: %v", err) @@ -158,12 +110,12 @@ func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) } func BenchmarkSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, false) } func BenchmarkConcurrentSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, true) } @@ -171,7 +123,7 @@ func BenchmarkMessageSizes(b *testing.B) { sizes := []int{100, 1024, 10240, 102400} for _, size := range sizes { b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, size, false) }) } diff --git a/transport/amqp/config.go b/transport/amqp/config.go index 4d705e5..fcf5c62 100644 --- a/transport/amqp/config.go +++ b/transport/amqp/config.go @@ -14,10 +14,9 @@ type OptionsConfig struct { } type PoolConfig struct { - Size int `yaml:"size" default:"10"` - MinSize int `yaml:"min_size" default:"5"` - MaxSize int `yaml:"max_size" default:"20"` - Dynamic bool `yaml:"dynamic" default:"false"` + Size int `yaml:"size" default:"10"` + MinSize int `yaml:"min_size" default:"5"` + MaxSize int `yaml:"max_size" default:"20"` } type ExchangeConfig struct { diff --git a/transport/amqp/consumer.go b/transport/amqp/consumer.go index 6e1ea2b..d844b76 100644 --- a/transport/amqp/consumer.go +++ b/transport/amqp/consumer.go @@ -74,45 +74,6 @@ func (c *Consumer) startWorkerPool( c.wg.Add(1) go c.startWorker(ctx, jobs, handler) } - - if c.cfg.Options.Pool.Dynamic { - go c.manageWorkerPool(ctx, jobs, handler) - } -} - -func (c *Consumer) manageWorkerPool( - ctx context.Context, - jobs chan job, - handler func(context.Context, api.Envelope) error, -) { - ticker := time.NewTicker(workerPoolCheckInterval) - defer ticker.Stop() - - currentSize := c.cfg.Options.Pool.Size - minSize := c.cfg.Options.Pool.MinSize - maxSize := c.cfg.Options.Pool.MaxSize - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if len(jobs) > currentSize && currentSize < maxSize { - toAdd := min(maxSize-currentSize, workerBatchSize) - - for range toAdd { - c.wg.Add(1) - go c.startWorker(ctx, jobs, handler) - } - - currentSize += toAdd - c.logger.DebugContext(ctx, "Increased worker pool size", "new_size", currentSize) - } else if len(jobs) == 0 && currentSize > minSize { - currentSize = max(currentSize-workerBatchSize, minSize) - c.logger.DebugContext(ctx, "Decreased worker pool size", "new_size", currentSize) - } - } - } } func (c *Consumer) startWorker( diff --git a/transport/chain.go b/transport/chain.go index dfb18ee..7ce3eb5 100644 --- a/transport/chain.go +++ b/transport/chain.go @@ -5,8 +5,9 @@ import ( "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type FactoryChain struct { diff --git a/transport/chain_test.go b/transport/chain_test.go index 13e8a98..032a0a8 100644 --- a/transport/chain_test.go +++ b/transport/chain_test.go @@ -6,13 +6,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/core/serializer" "github.com/gerfey/messenger/transport" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/helpers" ) diff --git a/transport/kafka/benchmark_test.go b/transport/kafka/benchmark_test.go index 2bb06c8..ebca1af 100644 --- a/transport/kafka/benchmark_test.go +++ b/transport/kafka/benchmark_test.go @@ -1,24 +1,18 @@ package kafka_test import ( - "context" - "errors" "fmt" "log/slog" - "os" "sync" - "sync/atomic" "testing" - "time" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" ) const ( - benchmarkDSN = "kafka://localhost:29092/" - benchmarkTimeout = 60 * time.Second + benchmarkDSN = "kafka://localhost:29092/" ) type BenchmarkMessage struct { @@ -27,34 +21,10 @@ type BenchmarkMessage struct { Data []byte } -func (m *BenchmarkMessage) RoutingKey() string { - return "benchmark_routing_key" -} - -type BenchmarkHandler struct { - processedCount *int64 - wg *sync.WaitGroup -} - -func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { - atomic.AddInt64(h.processedCount, 1) - if h.wg != nil { - h.wg.Done() - } - - return nil -} - -func (h *BenchmarkHandler) GetBusName() string { - return "default" -} - -func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { +func setupMessenger(b *testing.B) api.MessageBus { b.Helper() - logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ - Level: slog.LevelError, - })) + logger := slog.New(slog.DiscardHandler) cfg := &config.MessengerConfig{ DefaultBus: "default", @@ -66,10 +36,10 @@ func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { DSN: benchmarkDSN, Serializer: "default.transport.serializer", Options: map[string]any{ - "topics": []string{"benchmark_topic"}, - "group": "benchmark_group", + "topics": []string{"benchmark-topic"}, + "group": "benchmark-group", "producer": map[string]any{ - "async": false, + "async": true, }, }, }, @@ -79,37 +49,15 @@ func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { }, } - processedCount := int64(0) - var wg *sync.WaitGroup - if withWaitGroup { - wg = &sync.WaitGroup{} - } - - handler := &BenchmarkHandler{ - processedCount: &processedCount, - wg: wg, - } - builderInstance := builder.NewBuilder(cfg, logger) - if err := builderInstance.RegisterHandler(handler); err != nil { - b.Fatalf("Register handler failed: %v", err) - } + + builderInstance.RegisterMessage(&BenchmarkMessage{}) messenger, err := builderInstance.Build() if err != nil { b.Fatalf("Build messenger failed: %v", err) } - ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) - go func() { - defer cancel() - if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { - b.Logf("Messenger run error: %v", runErr) - } - }() - - time.Sleep(2 * time.Second) - bus, err := messenger.GetDefaultBus() if err != nil { b.Fatalf("Get default bus failed: %v", err) @@ -153,12 +101,12 @@ func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) } func BenchmarkSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, false) } func BenchmarkConcurrentSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, true) } @@ -166,7 +114,7 @@ func BenchmarkMessageSizes(b *testing.B) { sizes := []int{100, 1024, 10240, 102400} for _, size := range sizes { b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, size, false) }) } diff --git a/transport/kafka/config.go b/transport/kafka/config.go index 32c1392..9579c74 100644 --- a/transport/kafka/config.go +++ b/transport/kafka/config.go @@ -17,13 +17,14 @@ type OptionsConfig struct { } type ProducerOptionsConfig struct { - Async bool `yaml:"async" default:"false"` - RequiredAcks int `yaml:"required_acks" default:"1"` // 0, 1, -1 (all) - BatchSize int `yaml:"batch_size" default:"1000"` - BatchTimeout time.Duration `yaml:"batch_timeout" default:"20ms"` - WriteTimeout time.Duration `yaml:"write_timeout" default:"10s"` - ReadTimeout time.Duration `yaml:"read_timeout" default:"10s"` - Balancer string `yaml:"balancer" default:"least_bytes"` // least_bytes, hash, round_robin + Async bool `yaml:"async" default:"false"` + AutoTopicCreation bool `yaml:"auto_topic_creation" default:"false"` + RequiredAcks int `yaml:"required_acks" default:"1"` // 0, 1, -1 (all) + BatchSize int `yaml:"batch_size" default:"256"` + BatchTimeout time.Duration `yaml:"batch_timeout" default:"5ms"` + WriteTimeout time.Duration `yaml:"write_timeout" default:"10s"` + ReadTimeout time.Duration `yaml:"read_timeout" default:"10s"` + Balancer string `yaml:"balancer" default:"round_robin"` // least_bytes, hash, round_robin } type ConsumerOptionsConfig struct { @@ -47,10 +48,9 @@ type CommitConfig struct { } type PoolConfig struct { - Size int `yaml:"size" default:"3"` - MinSize int `yaml:"min_size" default:"2"` - MaxSize int `yaml:"max_size" default:"10"` - Dynamic bool `yaml:"dynamic" default:"true"` + Size int `yaml:"size" default:"3"` + MinSize int `yaml:"min_size" default:"2"` + MaxSize int `yaml:"max_size" default:"10"` } type RebalanceConfig struct { diff --git a/transport/kafka/connection.go b/transport/kafka/connection.go index 7600b4a..59c56d1 100644 --- a/transport/kafka/connection.go +++ b/transport/kafka/connection.go @@ -65,14 +65,15 @@ func (c *Connection) CreateWriter( balancer kafka.Balancer, ) *kafka.Writer { return &kafka.Writer{ - Addr: kafka.TCP(c.brokers...), - Topic: topic, - RequiredAcks: kafka.RequiredAcks(opts.RequiredAcks), - Async: async, - Balancer: balancer, - BatchSize: opts.BatchSize, - BatchTimeout: opts.BatchTimeout, - WriteTimeout: opts.WriteTimeout, - ReadTimeout: opts.ReadTimeout, + Addr: kafka.TCP(c.brokers...), + Topic: topic, + RequiredAcks: kafka.RequiredAcks(opts.RequiredAcks), + Async: async, + AllowAutoTopicCreation: opts.AutoTopicCreation, + Balancer: balancer, + BatchSize: opts.BatchSize, + BatchTimeout: opts.BatchTimeout, + WriteTimeout: opts.WriteTimeout, + ReadTimeout: opts.ReadTimeout, } } diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index 9ff7bf1..6ee9212 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -141,45 +141,6 @@ func (c *Consumer) startWorkerPool( c.wg.Add(1) go c.startWorker(ctx, jobs, handler) } - - if c.cfg.Options.Consumer.Pool.Dynamic { - go c.manageWorkerPool(ctx, jobs, handler) - } -} - -func (c *Consumer) manageWorkerPool( - ctx context.Context, - jobs chan job, - handler func(context.Context, api.Envelope) error, -) { - ticker := time.NewTicker(workerPoolCheckInterval) - defer ticker.Stop() - - currentSize := c.cfg.Options.Consumer.Pool.Size - minSize := c.cfg.Options.Consumer.Pool.MinSize - maxSize := c.cfg.Options.Consumer.Pool.MaxSize - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if len(jobs) > currentSize && currentSize < maxSize { - toAdd := min(maxSize-currentSize, workerBatchSize) - - for range toAdd { - c.wg.Add(1) - go c.startWorker(ctx, jobs, handler) - } - - currentSize += toAdd - c.logger.DebugContext(ctx, "Increased worker pool size", "new_size", currentSize) - } else if len(jobs) == 0 && currentSize > minSize { - currentSize = max(currentSize-workerBatchSize, minSize) - c.logger.DebugContext(ctx, "Decreased worker pool size", "new_size", currentSize) - } - } - } } func (c *Consumer) startWorker(ctx context.Context, jobs chan job, handler func(context.Context, api.Envelope) error) { diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index ad15df8..3d50d81 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -116,10 +116,6 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { if writeErr := writer.WriteMessages(ctx, msg); writeErr != nil { return fmt.Errorf("producer failed to write messages to topic %s: %w", topic, writeErr) } - - p.logger.DebugContext(ctx, "message sent to kafka topic", - slog.String("topic", topic), - slog.String("message_type", fmt.Sprintf("%T", env.Message()))) } return nil diff --git a/transport/redis/benchmark_test.go b/transport/redis/benchmark_test.go index 8c19d83..7c18d5e 100644 --- a/transport/redis/benchmark_test.go +++ b/transport/redis/benchmark_test.go @@ -1,24 +1,18 @@ package redis_test import ( - "context" - "errors" "fmt" "log/slog" - "os" "sync" - "sync/atomic" "testing" - "time" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" ) const ( - benchmarkDSN = "redis://localhost:6379/0" - benchmarkTimeout = 60 * time.Second + benchmarkDSN = "redis://localhost:6379/0" ) type BenchmarkMessage struct { @@ -27,34 +21,10 @@ type BenchmarkMessage struct { Data []byte } -func (m *BenchmarkMessage) RoutingKey() string { - return "benchmark_routing_key" -} - -type BenchmarkHandler struct { - processedCount *int64 - wg *sync.WaitGroup -} - -func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { - atomic.AddInt64(h.processedCount, 1) - if h.wg != nil { - h.wg.Done() - } - - return nil -} - -func (h *BenchmarkHandler) GetBusName() string { - return "default" -} - -func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { +func setupMessenger(b *testing.B) api.MessageBus { b.Helper() - logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ - Level: slog.LevelError, - })) + logger := slog.New(slog.DiscardHandler) cfg := &config.MessengerConfig{ DefaultBus: "default", @@ -78,37 +48,15 @@ func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { }, } - processedCount := int64(0) - var wg *sync.WaitGroup - if withWaitGroup { - wg = &sync.WaitGroup{} - } - - handler := &BenchmarkHandler{ - processedCount: &processedCount, - wg: wg, - } - builderInstance := builder.NewBuilder(cfg, logger) - if err := builderInstance.RegisterHandler(handler); err != nil { - b.Fatalf("Register handler failed: %v", err) - } + + builderInstance.RegisterMessage(&BenchmarkMessage{}) messenger, err := builderInstance.Build() if err != nil { b.Fatalf("Build messenger failed: %v", err) } - ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) - go func() { - defer cancel() - if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { - b.Logf("Messenger run error: %v", runErr) - } - }() - - time.Sleep(2 * time.Second) - bus, err := messenger.GetDefaultBus() if err != nil { b.Fatalf("Get default bus failed: %v", err) @@ -152,12 +100,12 @@ func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) } func BenchmarkSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, false) } func BenchmarkConcurrentSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, true) } @@ -165,7 +113,7 @@ func BenchmarkMessageSizes(b *testing.B) { sizes := []int{100, 1024, 10240, 102400} for _, size := range sizes { b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, size, false) }) } diff --git a/transport/sync/benchmark_test.go b/transport/sync/benchmark_test.go index 9eb1dd8..dc83195 100644 --- a/transport/sync/benchmark_test.go +++ b/transport/sync/benchmark_test.go @@ -1,24 +1,18 @@ package sync_test import ( - "context" - "errors" "fmt" "log/slog" - "os" "sync" - "sync/atomic" "testing" - "time" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" ) const ( - benchmarkDSN = "sync://" - benchmarkTimeout = 60 * time.Second + benchmarkDSN = "sync://" ) type BenchmarkMessage struct { @@ -27,34 +21,10 @@ type BenchmarkMessage struct { Data []byte } -func (m *BenchmarkMessage) RoutingKey() string { - return "benchmark_routing_key" -} - -type BenchmarkHandler struct { - processedCount *int64 - wg *sync.WaitGroup -} - -func (h *BenchmarkHandler) Handle(_ context.Context, _ *BenchmarkMessage) error { - atomic.AddInt64(h.processedCount, 1) - if h.wg != nil { - h.wg.Done() - } - - return nil -} - -func (h *BenchmarkHandler) GetBusName() string { - return "default" -} - -func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { +func setupMessenger(b *testing.B) api.MessageBus { b.Helper() - logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ - Level: slog.LevelError, - })) + logger := slog.New(slog.DiscardHandler) cfg := &config.MessengerConfig{ DefaultBus: "default", @@ -72,37 +42,15 @@ func setupMessenger(b *testing.B, withWaitGroup bool) api.MessageBus { }, } - processedCount := int64(0) - var wg *sync.WaitGroup - if withWaitGroup { - wg = &sync.WaitGroup{} - } - - handler := &BenchmarkHandler{ - processedCount: &processedCount, - wg: wg, - } - builderInstance := builder.NewBuilder(cfg, logger) - if err := builderInstance.RegisterHandler(handler); err != nil { - b.Fatalf("Register handler failed: %v", err) - } + + builderInstance.RegisterMessage(&BenchmarkMessage{}) messenger, err := builderInstance.Build() if err != nil { b.Fatalf("Build messenger failed: %v", err) } - ctx, cancel := context.WithTimeout(b.Context(), benchmarkTimeout) - go func() { - defer cancel() - if runErr := messenger.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) { - b.Logf("Messenger run error: %v", runErr) - } - }() - - time.Sleep(2 * time.Second) - bus, err := messenger.GetDefaultBus() if err != nil { b.Fatalf("Get default bus failed: %v", err) @@ -146,12 +94,12 @@ func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) } func BenchmarkSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, false) } func BenchmarkConcurrentSend(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, 100, true) } @@ -159,7 +107,7 @@ func BenchmarkMessageSizes(b *testing.B) { sizes := []int{100, 1024, 10240, 102400} for _, size := range sizes { b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { - bus := setupMessenger(b, false) + bus := setupMessenger(b) dispatchMessages(b, bus, size, false) }) } From e52b9c39811e7f92544abdb5d0a14bd47ebc54ec Mon Sep 17 00:00:00 2001 From: Gerfey Date: Wed, 13 Aug 2025 01:04:31 +0700 Subject: [PATCH 16/17] refactor: reorganizing the AMQP transport --- api/transport.go | 28 ++++- examples/messenger/messenger.go | 20 +++- transport/amqp/connection.go | 16 ++- transport/amqp/consumer.go | 32 +++--- transport/amqp/factory.go | 9 +- .../amqp/{base_publisher.go => producer.go} | 32 +++--- transport/amqp/publisher.go | 21 ---- transport/amqp/retry.go | 21 ---- transport/amqp/transport.go | 104 +++++++++--------- transport/manager.go | 10 +- 10 files changed, 155 insertions(+), 138 deletions(-) rename transport/amqp/{base_publisher.go => producer.go} (57%) delete mode 100644 transport/amqp/publisher.go delete mode 100644 transport/amqp/retry.go diff --git a/api/transport.go b/api/transport.go index 144e082..73c2e8b 100644 --- a/api/transport.go +++ b/api/transport.go @@ -8,6 +8,7 @@ import ( type Transport interface { Sender Receiver + Closer } type Sender interface { @@ -19,11 +20,34 @@ type Receiver interface { Receive(context.Context, func(context.Context, Envelope) error) error } +type Closer interface { + Close() error +} + +type Producer interface { + Send(context.Context, Envelope) error +} + +type Consumer interface { + Consume(context.Context, func(context.Context, Envelope) error) error +} + +type Connection interface { + Connect() error + IsConnect() bool + Close() error +} + type RetryableTransport interface { Transport Retry(context.Context, Envelope) error } +type SetupableTransport interface { + Transport + Setup(ctx context.Context) error +} + type SenderLocator interface { Register(string, Sender) error GetSenders(Envelope) []Sender @@ -39,7 +63,3 @@ type TransportFactory interface { type RoutedMessage interface { RoutingKey() string } - -type Setupable interface { - Setup(ctx context.Context) error -} diff --git a/examples/messenger/messenger.go b/examples/messenger/messenger.go index 3dd6978..98e5a7a 100644 --- a/examples/messenger/messenger.go +++ b/examples/messenger/messenger.go @@ -3,6 +3,9 @@ package main import ( "context" "log/slog" + "os" + "os/signal" + "syscall" "time" "github.com/gerfey/messenger/core/builder" @@ -13,11 +16,12 @@ import ( ) const ( - waitDurationSeconds = 20 + waitDurationSeconds = 5 ) func main() { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() log := slog.Default() @@ -41,6 +45,9 @@ func main() { return } + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + go func() { if runErr := messenger.Run(ctx); runErr != nil { log.Error("messenger run failed", "error", runErr) @@ -65,5 +72,14 @@ func main() { return } + select { + case sig := <-sigChan: + log.Info("Received signal, shutting down gracefully", "signal", sig) + cancel() + case <-time.After(waitDurationSeconds * time.Second): + log.Info("Timeout reached, shutting down") + cancel() + } + time.Sleep(waitDurationSeconds * time.Second) } diff --git a/transport/amqp/connection.go b/transport/amqp/connection.go index 9034fae..c6fb1d5 100644 --- a/transport/amqp/connection.go +++ b/transport/amqp/connection.go @@ -13,9 +13,9 @@ type Connection struct { lock sync.Mutex } -func NewConnection(dsn string) (*Connection, error) { +func NewConnection(dsn string) (ConnectionAMQP, error) { conn := &Connection{dsn: dsn} - err := conn.connect() + err := conn.Connect() if err != nil { return nil, err } @@ -28,7 +28,7 @@ func (c *Connection) Channel() (*amqp.Channel, error) { defer c.lock.Unlock() if c.conn == nil || c.conn.IsClosed() { - if err := c.connect(); err != nil { + if err := c.Connect(); err != nil { return nil, err } } @@ -41,7 +41,7 @@ func (c *Connection) Channel() (*amqp.Channel, error) { return ch, nil } -func (c *Connection) connect() error { +func (c *Connection) Connect() error { c.lock.Lock() defer c.lock.Unlock() @@ -54,3 +54,11 @@ func (c *Connection) connect() error { return nil } + +func (c *Connection) Close() error { + return c.conn.Close() +} + +func (c *Connection) IsConnect() bool { + return c.conn != nil && !c.conn.IsClosed() +} diff --git a/transport/amqp/consumer.go b/transport/amqp/consumer.go index d844b76..41b2e43 100644 --- a/transport/amqp/consumer.go +++ b/transport/amqp/consumer.go @@ -3,9 +3,7 @@ package amqp import ( "context" "fmt" - "log/slog" "sync" - "time" amqp "github.com/rabbitmq/amqp091-go" @@ -14,30 +12,30 @@ import ( ) const ( - defaultPoolSize = 10 - workerPoolCheckInterval = 30 * time.Second - workerBatchSize = 5 + defaultPoolSize = 10 ) type Consumer struct { - conn *Connection - cfg TransportConfig + config TransportConfig + connection ConnectionAMQP serializer api.Serializer wg sync.WaitGroup - logger *slog.Logger } -func NewConsumer(conn *Connection, cfg TransportConfig, serializer api.Serializer) *Consumer { +func NewConsumer(config TransportConfig, connection ConnectionAMQP, serializer api.Serializer) (api.Consumer, error) { return &Consumer{ - conn: conn, - cfg: cfg, + config: config, + connection: connection, serializer: serializer, - logger: slog.Default(), - } + }, nil } func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - ch, err := c.conn.Channel() + if !c.connection.IsConnect() { + return fmt.Errorf("amqp connection is not available") + } + + ch, err := c.connection.Channel() if err != nil { return fmt.Errorf("failed to create AMQP channel for consumer: %w", err) } @@ -65,7 +63,7 @@ func (c *Consumer) startWorkerPool( jobs chan job, handler func(context.Context, api.Envelope) error, ) { - poolSize := c.cfg.Options.Pool.Size + poolSize := c.config.Options.Pool.Size if poolSize <= 0 { poolSize = defaultPoolSize } @@ -97,7 +95,7 @@ func (c *Consumer) startWorker( } func (c *Consumer) startQueueConsumers(ctx context.Context, ch *amqp.Channel, jobs chan job) error { - for queueName := range c.cfg.Options.Queues { + for queueName := range c.config.Options.Queues { msgs, consumeErr := ch.ConsumeWithContext( ctx, queueName, @@ -152,7 +150,7 @@ func (c *Consumer) handleDelivery( } env = env.WithStamp(stamps.ReceivedStamp{ - Transport: c.cfg.Name, + Transport: c.config.Name, }) err = handler(ctx, env) diff --git a/transport/amqp/factory.go b/transport/amqp/factory.go index 4be2948..d6a1c78 100644 --- a/transport/amqp/factory.go +++ b/transport/amqp/factory.go @@ -25,7 +25,12 @@ func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "amqp://") } -func (f *TransportFactory) Create(name string, dsn string, options []byte, ser api.Serializer) (api.Transport, error) { +func (f *TransportFactory) Create( + name string, + dsn string, + options []byte, + serializer api.Serializer, +) (api.Transport, error) { var opts OptionsConfig if err := defaults.Set(&opts); err != nil { return nil, fmt.Errorf("set defaults: %w", err) @@ -41,5 +46,5 @@ func (f *TransportFactory) Create(name string, dsn string, options []byte, ser a Options: opts, } - return NewTransport(cfg, f.logger, ser) + return NewTransport(cfg, serializer) } diff --git a/transport/amqp/base_publisher.go b/transport/amqp/producer.go similarity index 57% rename from transport/amqp/base_publisher.go rename to transport/amqp/producer.go index 6959ea8..97caf2b 100644 --- a/transport/amqp/base_publisher.go +++ b/transport/amqp/producer.go @@ -3,28 +3,34 @@ package amqp import ( "context" "fmt" + "sync" amqp "github.com/rabbitmq/amqp091-go" "github.com/gerfey/messenger/api" ) -type BasePublisher struct { - conn *Connection - cfg TransportConfig +type Producer struct { + config TransportConfig + connection ConnectionAMQP serializer api.Serializer + lock sync.Mutex } -func NewBasePublisher(conn *Connection, cfg TransportConfig, serializer api.Serializer) *BasePublisher { - return &BasePublisher{ - conn: conn, - cfg: cfg, +func NewProducer(config TransportConfig, connection ConnectionAMQP, serializer api.Serializer) (api.Producer, error) { + return &Producer{ + config: config, + connection: connection, serializer: serializer, - } + }, nil } -func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) error { - body, headersMap, err := bp.serializer.Marshal(env) +func (p *Producer) Send(ctx context.Context, env api.Envelope) error { + if !p.connection.IsConnect() { + return fmt.Errorf("amqp connection is not available") + } + + body, headersMap, err := p.serializer.Marshal(env) if err != nil { return fmt.Errorf("failed to marshal envelope: %w", err) } @@ -34,7 +40,7 @@ func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) e headers[k] = v } - ch, err := bp.conn.Channel() + ch, err := p.connection.Channel() if err != nil { return fmt.Errorf("failed to create AMQP channel: %w", err) } @@ -45,7 +51,7 @@ func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) e routingKey := getRoutingKey(env.Message()) err = ch.PublishWithContext(ctx, - bp.cfg.Options.Exchange.Name, + p.config.Options.Exchange.Name, routingKey, false, false, @@ -57,7 +63,7 @@ func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) e if err != nil { return fmt.Errorf( "failed to publish message to exchange '%s' with routing key '%s': %w", - bp.cfg.Options.Exchange.Name, + p.config.Options.Exchange.Name, routingKey, err, ) diff --git a/transport/amqp/publisher.go b/transport/amqp/publisher.go deleted file mode 100644 index 3186835..0000000 --- a/transport/amqp/publisher.go +++ /dev/null @@ -1,21 +0,0 @@ -package amqp - -import ( - "context" - - "github.com/gerfey/messenger/api" -) - -type Publisher struct { - *BasePublisher -} - -func NewPublisher(conn *Connection, cfg TransportConfig, serializer api.Serializer) *Publisher { - return &Publisher{ - BasePublisher: NewBasePublisher(conn, cfg, serializer), - } -} - -func (p *Publisher) Publish(ctx context.Context, env api.Envelope) error { - return p.PublishMessage(ctx, env) -} diff --git a/transport/amqp/retry.go b/transport/amqp/retry.go deleted file mode 100644 index a2f79b3..0000000 --- a/transport/amqp/retry.go +++ /dev/null @@ -1,21 +0,0 @@ -package amqp - -import ( - "context" - - "github.com/gerfey/messenger/api" -) - -type Retry struct { - *BasePublisher -} - -func NewRetry(conn *Connection, cfg TransportConfig, serializer api.Serializer) *Retry { - return &Retry{ - BasePublisher: NewBasePublisher(conn, cfg, serializer), - } -} - -func (r *Retry) Retry(ctx context.Context, env api.Envelope) error { - return r.PublishMessage(ctx, env) -} diff --git a/transport/amqp/transport.go b/transport/amqp/transport.go index a2aa875..c81cf51 100644 --- a/transport/amqp/transport.go +++ b/transport/amqp/transport.go @@ -3,51 +3,60 @@ package amqp import ( "context" "fmt" - "log/slog" "reflect" + amqp "github.com/rabbitmq/amqp091-go" + "github.com/gerfey/messenger/api" ) +type ConnectionAMQP interface { + api.Connection + Channel() (*amqp.Channel, error) +} + type Transport struct { - cfg TransportConfig - publisher *Publisher - consumer *Consumer - retry *Retry + config TransportConfig + producer api.Producer + consumer api.Consumer + connection ConnectionAMQP serializer api.Serializer - conn *Connection - logger *slog.Logger } -func NewTransport(cfg TransportConfig, logger *slog.Logger, ser api.Serializer) (api.Transport, error) { - conn, err := NewConnection(cfg.DSN) - if err != nil { - logger.Error("failed to create AMQP connection", "dsn", cfg.DSN, "error", err) +func NewTransport( + config TransportConfig, + serializer api.Serializer, +) (api.Transport, error) { + connection, errConnection := NewConnection(config.DSN) + if errConnection != nil { + return nil, fmt.Errorf("failed to create connection: %w", errConnection) + } - return nil, err + producer, errProducer := NewProducer(config, connection, serializer) + if errProducer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errProducer) } - pub := NewPublisher(conn, cfg, ser) - cons := NewConsumer(conn, cfg, ser) - ret := NewRetry(conn, cfg, ser) + consumer, errConsumer := NewConsumer(config, connection, serializer) + if errConsumer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errConsumer) + } return &Transport{ - cfg: cfg, - publisher: pub, - consumer: cons, - retry: ret, - serializer: ser, - conn: conn, - logger: logger, + config: config, + producer: producer, + consumer: consumer, + connection: connection, + serializer: serializer, }, nil } func (t *Transport) Name() string { - return t.cfg.Name + return t.config.Name } func (t *Transport) Send(ctx context.Context, env api.Envelope) error { - return t.publisher.Publish(ctx, env) + return t.producer.Send(ctx, env) } func (t *Transport) Receive(ctx context.Context, handler func(context.Context, api.Envelope) error) error { @@ -55,18 +64,16 @@ func (t *Transport) Receive(ctx context.Context, handler func(context.Context, a } func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { - return t.retry.Retry(ctx, env) + return t.producer.Send(ctx, env) } -func (t *Transport) Setup(ctx context.Context) error { - if !t.cfg.Options.AutoSetup { +func (t *Transport) Setup(_ context.Context) error { + if !t.config.Options.AutoSetup { return nil } - ch, err := t.conn.Channel() + ch, err := t.connection.Channel() if err != nil { - t.logger.ErrorContext(ctx, "failed to open channel", "error", err) - return fmt.Errorf("failed to open channel: %w", err) } defer func() { @@ -74,21 +81,19 @@ func (t *Transport) Setup(ctx context.Context) error { }() err = ch.ExchangeDeclare( - t.cfg.Options.Exchange.Name, - t.cfg.Options.Exchange.Type, - t.cfg.Options.Exchange.Durable, - t.cfg.Options.Exchange.AutoDelete, - t.cfg.Options.Exchange.Internal, + t.config.Options.Exchange.Name, + t.config.Options.Exchange.Type, + t.config.Options.Exchange.Durable, + t.config.Options.Exchange.AutoDelete, + t.config.Options.Exchange.Internal, false, nil, ) if err != nil { - t.logger.ErrorContext(ctx, "failed to declare exchange", "exchange", t.cfg.Options.Exchange.Name, "error", err) - return fmt.Errorf("failed to declare exchange: %w", err) } - for queueName, queueCfg := range t.cfg.Options.Queues { + for queueName, queueCfg := range t.config.Options.Queues { _, err = ch.QueueDeclare( queueName, queueCfg.Durable, @@ -98,8 +103,6 @@ func (t *Transport) Setup(ctx context.Context) error { nil, ) if err != nil { - t.logger.ErrorContext(ctx, "declare queue", "queue", queueName, "error", err) - return fmt.Errorf("declare queue: %w", err) } @@ -113,22 +116,11 @@ func (t *Transport) Setup(ctx context.Context) error { bindErr := ch.QueueBind( queueName, bindingKey, - t.cfg.Options.Exchange.Name, + t.config.Options.Exchange.Name, false, nil, ) if bindErr != nil { - t.logger.ErrorContext( - ctx, - "bind queue", - "queue", - queueName, - "binding_key", - bindingKey, - "error", - bindErr, - ) - return fmt.Errorf("bind queue: %w", bindErr) } } @@ -137,6 +129,14 @@ func (t *Transport) Setup(ctx context.Context) error { return nil } +func (t *Transport) Close() error { + if err := t.connection.Close(); err != nil { + return fmt.Errorf("failed to close connection: %w", err) + } + + return nil +} + func getRoutingKey(msg any) string { var routingKey string if rk, ok := msg.(api.RoutedMessage); ok { diff --git a/transport/manager.go b/transport/manager.go index 4412921..ec1274a 100644 --- a/transport/manager.go +++ b/transport/manager.go @@ -53,7 +53,7 @@ func (m *Manager) Start(ctx context.Context, consumeOnly []string) { m.running = true for _, t := range m.transports { - if s, ok := t.(api.Setupable); ok { + if s, ok := t.(api.SetupableTransport); ok { if err := s.Setup(ctx); err != nil { m.logger.ErrorContext(ctx, "failed to setup transport", "name", t.Name(), "error", err) @@ -73,6 +73,12 @@ func (m *Manager) Stop() { m.running = false m.mu.Unlock() + for _, t := range m.transports { + if errClose := t.Close(); errClose != nil { + m.logger.Error("failed to close transport", "name", t.Name(), "error", errClose) + } + } + m.wg.Wait() } @@ -137,7 +143,7 @@ func (m *Manager) receiveTransport(ctx context.Context, t api.Transport) { }) if err != nil { - m.logger.Error("receive error", "error", err) + m.logger.ErrorContext(ctx, "receive error", "transport", t.Name(), "error", err) } }(t) } From bca9d06211ce30ae83afef8be7d015290abb36cd Mon Sep 17 00:00:00 2001 From: Gerfey Date: Thu, 14 Aug 2025 15:52:40 +0700 Subject: [PATCH 17/17] refactor: unify transport interfaces --- api/transport.go | 6 +- core/builder/builder.go | 13 +- tests/helpers/messages.go | 2 +- tests/mocks/mock_transport.go | 272 ++++++++++++++++++++++++++++- transport/amqp/consumer.go | 7 +- transport/amqp/factory.go | 11 +- transport/amqp/factory_test.go | 11 +- transport/amqp/producer.go | 9 +- transport/amqp/transport.go | 6 +- transport/inmemory/factory.go | 11 +- transport/inmemory/factory_test.go | 10 +- transport/inmemory/transport.go | 9 + transport/kafka/connection.go | 40 ++--- transport/kafka/consumer.go | 97 ++++------ transport/kafka/factory.go | 13 +- transport/kafka/factory_test.go | 11 +- transport/kafka/producer.go | 69 ++++---- transport/kafka/transport.go | 41 +++-- transport/redis/consumer.go | 77 ++++---- transport/redis/factory.go | 20 +-- transport/redis/producer.go | 27 ++- transport/redis/transport.go | 44 +++-- transport/sync/transport.go | 4 + 23 files changed, 516 insertions(+), 294 deletions(-) diff --git a/api/transport.go b/api/transport.go index 73c2e8b..242acde 100644 --- a/api/transport.go +++ b/api/transport.go @@ -26,15 +26,11 @@ type Closer interface { type Producer interface { Send(context.Context, Envelope) error + Close() error } type Consumer interface { Consume(context.Context, func(context.Context, Envelope) error) error -} - -type Connection interface { - Connect() error - IsConnect() bool Close() error } diff --git a/core/builder/builder.go b/core/builder/builder.go index e36b7a8..e80e761 100644 --- a/core/builder/builder.go +++ b/core/builder/builder.go @@ -42,22 +42,21 @@ type Builder struct { func NewBuilder(cfg *config.MessengerConfig, logger *slog.Logger) api.Builder { resolver := NewResolver() - busLocator := bus.NewLocator() serializerLocator := serializer.NewSerializerLocator() - tf := transport.NewFactoryChain( - amqp.NewTransportFactory(logger), - inmemory.NewTransportFactory(logger), + transportFactory := transport.NewFactoryChain( sync.NewTransportFactory(busLocator), - kafka.NewTransportFactory(logger), - redis.NewTransportFactory(logger), + inmemory.NewTransportFactory(), + amqp.NewTransportFactory(), + kafka.NewTransportFactory(), + redis.NewTransportFactory(), ) return &Builder{ cfg: cfg, resolver: resolver, - transportFactory: tf, + transportFactory: transportFactory, handlersLocator: handler.NewHandlerLocator(), senderLocator: transport.NewSenderLocator(), middlewareLocator: middleware.NewMiddlewareLocator(), diff --git a/tests/helpers/messages.go b/tests/helpers/messages.go index 67bb43e..e4555bf 100644 --- a/tests/helpers/messages.go +++ b/tests/helpers/messages.go @@ -200,7 +200,7 @@ func (t *TestTransport) Start(_ context.Context) error { return nil } -func (t *TestTransport) Stop() error { +func (t *TestTransport) Close() error { t.IsStopped = true return nil diff --git a/tests/mocks/mock_transport.go b/tests/mocks/mock_transport.go index 3d69a78..6ed59ae 100644 --- a/tests/mocks/mock_transport.go +++ b/tests/mocks/mock_transport.go @@ -41,6 +41,20 @@ func (m *MockTransport) EXPECT() *MockTransportMockRecorder { return m.recorder } +// Close mocks base method. +func (m *MockTransport) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockTransportMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTransport)(nil).Close)) +} + // Name mocks base method. func (m *MockTransport) Name() string { m.ctrl.T.Helper() @@ -173,6 +187,148 @@ func (mr *MockReceiverMockRecorder) Receive(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Receive", reflect.TypeOf((*MockReceiver)(nil).Receive), arg0, arg1) } +// MockCloser is a mock of Closer interface. +type MockCloser struct { + ctrl *gomock.Controller + recorder *MockCloserMockRecorder + isgomock struct{} +} + +// MockCloserMockRecorder is the mock recorder for MockCloser. +type MockCloserMockRecorder struct { + mock *MockCloser +} + +// NewMockCloser creates a new mock instance. +func NewMockCloser(ctrl *gomock.Controller) *MockCloser { + mock := &MockCloser{ctrl: ctrl} + mock.recorder = &MockCloserMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCloser) EXPECT() *MockCloserMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockCloser) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockCloserMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockCloser)(nil).Close)) +} + +// MockProducer is a mock of Producer interface. +type MockProducer struct { + ctrl *gomock.Controller + recorder *MockProducerMockRecorder + isgomock struct{} +} + +// MockProducerMockRecorder is the mock recorder for MockProducer. +type MockProducerMockRecorder struct { + mock *MockProducer +} + +// NewMockProducer creates a new mock instance. +func NewMockProducer(ctrl *gomock.Controller) *MockProducer { + mock := &MockProducer{ctrl: ctrl} + mock.recorder = &MockProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProducer) EXPECT() *MockProducerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockProducer) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockProducer)(nil).Close)) +} + +// Send mocks base method. +func (m *MockProducer) Send(arg0 context.Context, arg1 api.Envelope) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockProducerMockRecorder) Send(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockProducer)(nil).Send), arg0, arg1) +} + +// MockConsumer is a mock of Consumer interface. +type MockConsumer struct { + ctrl *gomock.Controller + recorder *MockConsumerMockRecorder + isgomock struct{} +} + +// MockConsumerMockRecorder is the mock recorder for MockConsumer. +type MockConsumerMockRecorder struct { + mock *MockConsumer +} + +// NewMockConsumer creates a new mock instance. +func NewMockConsumer(ctrl *gomock.Controller) *MockConsumer { + mock := &MockConsumer{ctrl: ctrl} + mock.recorder = &MockConsumerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConsumer) EXPECT() *MockConsumerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockConsumer) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockConsumerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockConsumer)(nil).Close)) +} + +// Consume mocks base method. +func (m *MockConsumer) Consume(arg0 context.Context, arg1 func(context.Context, api.Envelope) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Consume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Consume indicates an expected call of Consume. +func (mr *MockConsumerMockRecorder) Consume(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Consume", reflect.TypeOf((*MockConsumer)(nil).Consume), arg0, arg1) +} + // MockRetryableTransport is a mock of RetryableTransport interface. type MockRetryableTransport struct { ctrl *gomock.Controller @@ -197,6 +353,20 @@ func (m *MockRetryableTransport) EXPECT() *MockRetryableTransportMockRecorder { return m.recorder } +// Close mocks base method. +func (m *MockRetryableTransport) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockRetryableTransportMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockRetryableTransport)(nil).Close)) +} + // Name mocks base method. func (m *MockRetryableTransport) Name() string { m.ctrl.T.Helper() @@ -253,6 +423,100 @@ func (mr *MockRetryableTransportMockRecorder) Send(arg0, arg1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockRetryableTransport)(nil).Send), arg0, arg1) } +// MockSetupableTransport is a mock of SetupableTransport interface. +type MockSetupableTransport struct { + ctrl *gomock.Controller + recorder *MockSetupableTransportMockRecorder + isgomock struct{} +} + +// MockSetupableTransportMockRecorder is the mock recorder for MockSetupableTransport. +type MockSetupableTransportMockRecorder struct { + mock *MockSetupableTransport +} + +// NewMockSetupableTransport creates a new mock instance. +func NewMockSetupableTransport(ctrl *gomock.Controller) *MockSetupableTransport { + mock := &MockSetupableTransport{ctrl: ctrl} + mock.recorder = &MockSetupableTransportMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSetupableTransport) EXPECT() *MockSetupableTransportMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockSetupableTransport) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockSetupableTransportMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSetupableTransport)(nil).Close)) +} + +// Name mocks base method. +func (m *MockSetupableTransport) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockSetupableTransportMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockSetupableTransport)(nil).Name)) +} + +// Receive mocks base method. +func (m *MockSetupableTransport) Receive(arg0 context.Context, arg1 func(context.Context, api.Envelope) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Receive", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Receive indicates an expected call of Receive. +func (mr *MockSetupableTransportMockRecorder) Receive(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Receive", reflect.TypeOf((*MockSetupableTransport)(nil).Receive), arg0, arg1) +} + +// Send mocks base method. +func (m *MockSetupableTransport) Send(arg0 context.Context, arg1 api.Envelope) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockSetupableTransportMockRecorder) Send(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSetupableTransport)(nil).Send), arg0, arg1) +} + +// Setup mocks base method. +func (m *MockSetupableTransport) Setup(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Setup", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Setup indicates an expected call of Setup. +func (mr *MockSetupableTransportMockRecorder) Setup(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Setup", reflect.TypeOf((*MockSetupableTransport)(nil).Setup), ctx) +} + // MockSenderLocator is a mock of SenderLocator interface. type MockSenderLocator struct { ctrl *gomock.Controller @@ -354,18 +618,18 @@ func (m *MockTransportFactory) EXPECT() *MockTransportFactoryMockRecorder { } // Create mocks base method. -func (m *MockTransportFactory) Create(arg0, arg1 string, arg2 []byte) (api.Transport, error) { +func (m *MockTransportFactory) Create(arg0, arg1 string, arg2 []byte, arg3 api.Serializer) (api.Transport, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(api.Transport) ret1, _ := ret[1].(error) return ret0, ret1 } // Create indicates an expected call of Create. -func (mr *MockTransportFactoryMockRecorder) Create(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockTransportFactoryMockRecorder) Create(arg0, arg1, arg2, arg3 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockTransportFactory)(nil).Create), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockTransportFactory)(nil).Create), arg0, arg1, arg2, arg3) } // Supports mocks base method. diff --git a/transport/amqp/consumer.go b/transport/amqp/consumer.go index 41b2e43..b7006c7 100644 --- a/transport/amqp/consumer.go +++ b/transport/amqp/consumer.go @@ -2,6 +2,7 @@ package amqp import ( "context" + "errors" "fmt" "sync" @@ -32,7 +33,7 @@ func NewConsumer(config TransportConfig, connection ConnectionAMQP, serializer a func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { if !c.connection.IsConnect() { - return fmt.Errorf("amqp connection is not available") + return errors.New("amqp connection is not available") } ch, err := c.connection.Channel() @@ -58,6 +59,10 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap return ctx.Err() } +func (c *Consumer) Close() error { + return nil +} + func (c *Consumer) startWorkerPool( ctx context.Context, jobs chan job, diff --git a/transport/amqp/factory.go b/transport/amqp/factory.go index d6a1c78..1cb0275 100644 --- a/transport/amqp/factory.go +++ b/transport/amqp/factory.go @@ -2,7 +2,6 @@ package amqp import ( "fmt" - "log/slog" "strings" "github.com/creasty/defaults" @@ -11,14 +10,10 @@ import ( "github.com/gerfey/messenger/api" ) -type TransportFactory struct { - logger *slog.Logger -} +type TransportFactory struct{} -func NewTransportFactory(logger *slog.Logger) api.TransportFactory { - return &TransportFactory{ - logger: logger, - } +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} } func (f *TransportFactory) Supports(dsn string) bool { diff --git a/transport/amqp/factory_test.go b/transport/amqp/factory_test.go index 12a572d..d21aad6 100644 --- a/transport/amqp/factory_test.go +++ b/transport/amqp/factory_test.go @@ -1,7 +1,6 @@ package amqp_test import ( - "log/slog" "testing" "github.com/stretchr/testify/assert" @@ -19,9 +18,7 @@ func TestNewTransportFactory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - - factory := amqp.NewTransportFactory(logger) + factory := amqp.NewTransportFactory() assert.NotNil(t, factory) assert.IsType(t, &amqp.TransportFactory{}, factory) @@ -65,8 +62,7 @@ func TestTransportFactory_Supports(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - factory := amqp.NewTransportFactory(logger) + factory := amqp.NewTransportFactory() got := factory.Supports(tt.dsn) assert.Equal(t, tt.want, got) @@ -78,9 +74,8 @@ func TestTransportFactory_Create(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := amqp.NewTransportFactory(logger) + factory := amqp.NewTransportFactory() name := "test-amqp" diff --git a/transport/amqp/producer.go b/transport/amqp/producer.go index 97caf2b..58eca9f 100644 --- a/transport/amqp/producer.go +++ b/transport/amqp/producer.go @@ -2,8 +2,8 @@ package amqp import ( "context" + "errors" "fmt" - "sync" amqp "github.com/rabbitmq/amqp091-go" @@ -14,7 +14,6 @@ type Producer struct { config TransportConfig connection ConnectionAMQP serializer api.Serializer - lock sync.Mutex } func NewProducer(config TransportConfig, connection ConnectionAMQP, serializer api.Serializer) (api.Producer, error) { @@ -27,7 +26,7 @@ func NewProducer(config TransportConfig, connection ConnectionAMQP, serializer a func (p *Producer) Send(ctx context.Context, env api.Envelope) error { if !p.connection.IsConnect() { - return fmt.Errorf("amqp connection is not available") + return errors.New("amqp connection is not available") } body, headersMap, err := p.serializer.Marshal(env) @@ -71,3 +70,7 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { return nil } + +func (p *Producer) Close() error { + return nil +} diff --git a/transport/amqp/transport.go b/transport/amqp/transport.go index c81cf51..cf5ad75 100644 --- a/transport/amqp/transport.go +++ b/transport/amqp/transport.go @@ -11,8 +11,10 @@ import ( ) type ConnectionAMQP interface { - api.Connection Channel() (*amqp.Channel, error) + Connect() error + IsConnect() bool + Close() error } type Transport struct { @@ -20,7 +22,6 @@ type Transport struct { producer api.Producer consumer api.Consumer connection ConnectionAMQP - serializer api.Serializer } func NewTransport( @@ -47,7 +48,6 @@ func NewTransport( producer: producer, consumer: consumer, connection: connection, - serializer: serializer, }, nil } diff --git a/transport/inmemory/factory.go b/transport/inmemory/factory.go index 0af5f98..3f11590 100644 --- a/transport/inmemory/factory.go +++ b/transport/inmemory/factory.go @@ -1,20 +1,15 @@ package inmemory import ( - "log/slog" "strings" "github.com/gerfey/messenger/api" ) -type TransportFactory struct { - logger *slog.Logger -} +type TransportFactory struct{} -func NewTransportFactory(logger *slog.Logger) api.TransportFactory { - return &TransportFactory{ - logger: logger, - } +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} } func (f *TransportFactory) Supports(dsn string) bool { diff --git a/transport/inmemory/factory_test.go b/transport/inmemory/factory_test.go index cacfe6d..90914d3 100644 --- a/transport/inmemory/factory_test.go +++ b/transport/inmemory/factory_test.go @@ -1,7 +1,6 @@ package inmemory_test import ( - "log/slog" "testing" "github.com/stretchr/testify/assert" @@ -19,8 +18,7 @@ func TestNewTransportFactory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - factory := inmemory.NewTransportFactory(logger) + factory := inmemory.NewTransportFactory() assert.NotNil(t, factory) assert.IsType(t, &inmemory.TransportFactory{}, factory) @@ -30,8 +28,7 @@ func TestTransportFactory_Supports(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - factory := inmemory.NewTransportFactory(logger) + factory := inmemory.NewTransportFactory() testCases := []struct { name string @@ -72,9 +69,8 @@ func TestTransportFactory_Create(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := inmemory.NewTransportFactory(logger) + factory := inmemory.NewTransportFactory() name := "test-inmemory" dsn := "in-memory://test" diff --git a/transport/inmemory/transport.go b/transport/inmemory/transport.go index 404f910..5e5edc8 100644 --- a/transport/inmemory/transport.go +++ b/transport/inmemory/transport.go @@ -64,3 +64,12 @@ func (t *Transport) Receive(ctx context.Context, handler func(context.Context, a } } } + +func (t *Transport) Close() error { + t.lock.Lock() + defer t.lock.Unlock() + + t.queue = nil + + return nil +} diff --git a/transport/kafka/connection.go b/transport/kafka/connection.go index 59c56d1..577069c 100644 --- a/transport/kafka/connection.go +++ b/transport/kafka/connection.go @@ -17,7 +17,7 @@ type Connection struct { dialer *kafka.Dialer } -func NewConnection(brokers []string) (*Connection, error) { +func NewConnection(brokers []string) (ConnectionKafka, error) { conn := &Connection{ brokers: brokers, dialer: &kafka.Dialer{ @@ -26,31 +26,13 @@ func NewConnection(brokers []string) (*Connection, error) { }, } - if err := conn.Check(connectionTimeout); err != nil { + if err := conn.check(connectionTimeout); err != nil { return nil, err } return conn, nil } -func (c *Connection) Check(timeout time.Duration) error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - for _, broker := range c.brokers { - conn, connErr := c.dialer.DialContext(ctx, "tcp", broker) - if connErr != nil { - return fmt.Errorf("failed to connect to Kafka broker at '%s': %w", broker, connErr) - } - - if closeErr := conn.Close(); closeErr != nil { - return fmt.Errorf("failed to close connection to Kafka broker at '%s': %w", broker, closeErr) - } - } - - return nil -} - func (c *Connection) CreateReader(config kafka.ReaderConfig) *kafka.Reader { config.Brokers = c.brokers config.Dialer = c.dialer @@ -77,3 +59,21 @@ func (c *Connection) CreateWriter( ReadTimeout: opts.ReadTimeout, } } + +func (c *Connection) check(timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + for _, broker := range c.brokers { + conn, connErr := c.dialer.DialContext(ctx, "tcp", broker) + if connErr != nil { + return fmt.Errorf("failed to connect to Kafka broker at '%s': %w", broker, connErr) + } + + if closeErr := conn.Close(); closeErr != nil { + return fmt.Errorf("failed to close connection to Kafka broker at '%s': %w", broker, closeErr) + } + } + + return nil +} diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go index 6ee9212..1789e31 100644 --- a/transport/kafka/consumer.go +++ b/transport/kafka/consumer.go @@ -3,7 +3,6 @@ package kafka import ( "context" "fmt" - "log/slog" "sync" "time" @@ -16,29 +15,21 @@ import ( const ( minBytes = 10e3 // 10KB maxBytes = 10e6 // 10MB - sessionTimeout = 10 * time.Second rebalanceTimeout = 5 * time.Second - heartbeatInterval = 2 * time.Second defaultPoolSize = 10 readLagInterval = -1 - - workerPoolCheckInterval = 30 * time.Second - workerBatchSize = 5 - errorBackoffDelay = 100 * time.Millisecond + errorBackoffDelay = 100 * time.Millisecond ) type Consumer struct { - cfg TransportConfig + config TransportConfig serializer api.Serializer - conn *Connection + connection ConnectionKafka readers []*kafka.Reader wg sync.WaitGroup - ctx context.Context - cancel context.CancelFunc batchMutex sync.Mutex batchMessages []kafka.Message deferredCommits sync.Map - logger *slog.Logger } type messageWithReader struct { @@ -46,36 +37,31 @@ type messageWithReader struct { reader *kafka.Reader } -func NewConsumer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) *Consumer { - ctx, cancel := context.WithCancel(context.Background()) - +func NewConsumer(config TransportConfig, connection ConnectionKafka, serializer api.Serializer) (api.Consumer, error) { return &Consumer{ - cfg: cfg, - serializer: ser, - conn: conn, + config: config, + connection: connection, + serializer: serializer, readers: make([]*kafka.Reader, 0), - ctx: ctx, - cancel: cancel, - logger: logger, - } + }, nil } func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - for _, topic := range c.cfg.Options.Topics { + for _, topic := range c.config.Options.Topics { readerConfig := kafka.ReaderConfig{ - GroupID: c.cfg.Options.Group, + GroupID: c.config.Options.Group, Topic: topic, - CommitInterval: c.cfg.Options.Consumer.Commit.Interval, + CommitInterval: c.config.Options.Consumer.Commit.Interval, MinBytes: minBytes, MaxBytes: maxBytes, ReadLagInterval: readLagInterval, - SessionTimeout: c.cfg.Options.Consumer.SessionTimeout, + SessionTimeout: c.config.Options.Consumer.SessionTimeout, RebalanceTimeout: rebalanceTimeout, - HeartbeatInterval: c.cfg.Options.Consumer.HeartbeatInterval, + HeartbeatInterval: c.config.Options.Consumer.HeartbeatInterval, MaxWait: time.Second, } - switch c.cfg.Options.Consumer.Rebalance.Strategy { + switch c.config.Options.Consumer.Rebalance.Strategy { case "range": readerConfig.GroupBalancers = []kafka.GroupBalancer{kafka.RangeGroupBalancer{}} case "roundrobin": @@ -83,7 +69,7 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap } c.configureOffset(&readerConfig) - reader := c.conn.CreateReader(readerConfig) + reader := c.connection.CreateReader(readerConfig) c.readers = append(c.readers, reader) } @@ -98,30 +84,35 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap }(reader) } - if c.cfg.Options.Consumer.Commit.Strategy == "batch" { + if c.config.Options.Consumer.Commit.Strategy == "batch" { go c.startBatchCommitter(ctx) } <-ctx.Done() + close(jobs) + c.wg.Wait() + + return ctx.Err() +} + +func (c *Consumer) Close() error { for _, reader := range c.readers { - if err := reader.Close(); err != nil { - c.logger.ErrorContext(ctx, "Failed to close Kafka reader", "error", err) + err := reader.Close() + if err != nil { + return err } } - c.cancel() - c.wg.Wait() - - return ctx.Err() + return nil } func (c *Consumer) configureOffset(config *kafka.ReaderConfig) { - switch c.cfg.Options.Consumer.OffsetConfig.Type { + switch c.config.Options.Consumer.OffsetConfig.Type { case "earliest": config.StartOffset = kafka.FirstOffset case "specific": - config.StartOffset = c.cfg.Options.Consumer.OffsetConfig.Value + config.StartOffset = c.config.Options.Consumer.OffsetConfig.Value default: config.StartOffset = kafka.LastOffset } @@ -132,7 +123,7 @@ func (c *Consumer) startWorkerPool( jobs chan job, handler func(context.Context, api.Envelope) error, ) { - poolSize := c.cfg.Options.Consumer.Pool.Size + poolSize := c.config.Options.Consumer.Pool.Size if poolSize <= 0 { poolSize = defaultPoolSize } @@ -171,7 +162,6 @@ func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan return } - c.logger.ErrorContext(ctx, "Failed to fetch message", "error", err) time.Sleep(errorBackoffDelay) continue @@ -194,17 +184,14 @@ func (c *Consumer) handleMessage( ) { env, err := c.serializer.Unmarshal(msg.Value, c.headerMap(msg.Headers)) if err != nil { - c.logger.ErrorContext(ctx, "Failed to unmarshal message", "error", err) - c.commitMessage(ctx, r, msg) return } - env = env.WithStamp(stamps.ReceivedStamp{Transport: c.cfg.Name}) + env = env.WithStamp(stamps.ReceivedStamp{Transport: c.config.Name}) if handlerErr := handler(ctx, env); handlerErr != nil { - c.logger.ErrorContext(ctx, "Handler failed", "error", handlerErr) c.commitMessage(ctx, r, msg) return @@ -214,15 +201,9 @@ func (c *Consumer) handleMessage( } func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka.Message) { - switch c.cfg.Options.Consumer.Commit.Strategy { + switch c.config.Options.Consumer.Commit.Strategy { case "auto": - if err := r.CommitMessages(ctx, msg); err != nil { - c.logger.ErrorContext(ctx, "Failed to commit message", - "topic", msg.Topic, - "partition", msg.Partition, - "offset", msg.Offset, - "error", err) - } + _ = r.CommitMessages(ctx, msg) case "manual": case "batch": c.batchMutex.Lock() @@ -231,7 +212,7 @@ func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka msgKey := fmt.Sprintf("%s-%d-%d", msg.Topic, msg.Partition, msg.Offset) c.deferredCommits.Store(msgKey, messageWithReader{message: msg, reader: r}) - if len(c.batchMessages) >= c.cfg.Options.Consumer.Commit.BatchSize { + if len(c.batchMessages) >= c.config.Options.Consumer.Commit.BatchSize { c.batchMutex.Unlock() c.commitBatch(ctx) } else { @@ -244,7 +225,7 @@ func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka } func (c *Consumer) startBatchCommitter(ctx context.Context) { - ticker := time.NewTicker(c.cfg.Options.Consumer.Commit.Interval) + ticker := time.NewTicker(c.config.Options.Consumer.Commit.Interval) defer ticker.Stop() for { @@ -310,13 +291,7 @@ func (c *Consumer) commitMessagesAndCleanup( partitionOffsets map[int]kafka.Message, ) { for _, msg := range partitionOffsets { - if err := reader.CommitMessages(ctx, msg); err != nil { - c.logger.ErrorContext(ctx, "Failed to commit message batch", - "topic", msg.Topic, - "partition", msg.Partition, - "offset", msg.Offset, - "error", err.Error()) - } else { + if err := reader.CommitMessages(ctx, msg); err == nil { c.cleanupCommittedMessages(messages, msg) } } diff --git a/transport/kafka/factory.go b/transport/kafka/factory.go index 94cd7ce..732d6a9 100644 --- a/transport/kafka/factory.go +++ b/transport/kafka/factory.go @@ -2,7 +2,6 @@ package kafka import ( "fmt" - "log/slog" "strings" "github.com/creasty/defaults" @@ -11,14 +10,10 @@ import ( "github.com/gerfey/messenger/api" ) -type TransportFactory struct { - logger *slog.Logger -} +type TransportFactory struct{} -func NewTransportFactory(logger *slog.Logger) api.TransportFactory { - return &TransportFactory{ - logger: logger, - } +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} } func (t *TransportFactory) Supports(dsn string) bool { @@ -41,5 +36,5 @@ func (t *TransportFactory) Create(name string, dsn string, options []byte, ser a Options: optsConfig, } - return NewTransport(tCfg, t.logger, ser) + return NewTransport(tCfg, ser) } diff --git a/transport/kafka/factory_test.go b/transport/kafka/factory_test.go index b05f454..f5be521 100644 --- a/transport/kafka/factory_test.go +++ b/transport/kafka/factory_test.go @@ -1,7 +1,6 @@ package kafka_test import ( - "log/slog" "testing" "github.com/stretchr/testify/assert" @@ -19,9 +18,7 @@ func TestNewTransportFactory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - - factory := kafka.NewTransportFactory(logger) + factory := kafka.NewTransportFactory() assert.NotNil(t, factory) assert.IsType(t, &kafka.TransportFactory{}, factory) @@ -65,8 +62,7 @@ func TestTransportFactory_Supports(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - factory := kafka.NewTransportFactory(logger) + factory := kafka.NewTransportFactory() result := factory.Supports(tc.dsn) assert.Equal(t, tc.expected, result) @@ -78,9 +74,8 @@ func TestTransportFactory_Create(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := kafka.NewTransportFactory(logger) + factory := kafka.NewTransportFactory() name := "test-kafka" dsn := "kafka://non-existent-host:9092" diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go index 3d50d81..f68aefc 100644 --- a/transport/kafka/producer.go +++ b/transport/kafka/producer.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "log/slog" "sync" "time" @@ -15,29 +14,27 @@ import ( ) type Producer struct { - cfg TransportConfig + config TransportConfig serializer api.Serializer - conn *Connection - logger *slog.Logger + connection ConnectionKafka writers map[string]*kafka.Writer mu sync.RWMutex } -func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) (*Producer, error) { +func NewProducer(config TransportConfig, connection ConnectionKafka, serializer api.Serializer) (api.Producer, error) { p := &Producer{ - cfg: cfg, - serializer: ser, - conn: conn, - logger: logger, + config: config, + connection: connection, + serializer: serializer, writers: make(map[string]*kafka.Writer), } - if len(cfg.Options.Topics) == 0 { + if len(config.Options.Topics) == 0 { return nil, errors.New("no topics configured for kafka transport") } var balancer kafka.Balancer = &kafka.LeastBytes{} - switch cfg.Options.Producer.Balancer { + switch config.Options.Producer.Balancer { case "hash": balancer = &kafka.Hash{} case "round_robin": @@ -46,15 +43,15 @@ func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection, logg balancer = &kafka.LeastBytes{} } - if cfg.Options.Key.Strategy != "none" { + if config.Options.Key.Strategy != "none" { balancer = &kafka.Hash{} } - for _, topic := range cfg.Options.Topics { - writer := conn.CreateWriter( + for _, topic := range config.Options.Topics { + writer := connection.CreateWriter( topic, - cfg.Options.Producer, - cfg.Options.Producer.Async, + config.Options.Producer, + config.Options.Producer.Async, balancer, ) @@ -64,24 +61,6 @@ func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection, logg return p, nil } -func (p *Producer) Close() error { - p.mu.Lock() - defer p.mu.Unlock() - - var errs []error - for topic, writer := range p.writers { - if err := writer.Close(); err != nil { - errs = append(errs, fmt.Errorf("failed to close writer for topic %s: %w", topic, err)) - } - } - - if len(errs) > 0 { - return fmt.Errorf("errors closing writers: %v", errs) - } - - return nil -} - func (p *Producer) Send(ctx context.Context, env api.Envelope) error { payload, headers, err := p.serializer.Marshal(env) if err != nil { @@ -107,7 +86,7 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { p.mu.RLock() defer p.mu.RUnlock() - for _, topic := range p.cfg.Options.Topics { + for _, topic := range p.config.Options.Topics { writer, exists := p.writers[topic] if !exists { return fmt.Errorf("writer for topic %s not found", topic) @@ -121,8 +100,26 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { return nil } +func (p *Producer) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + + var errs []error + for topic, writer := range p.writers { + if err := writer.Close(); err != nil { + errs = append(errs, fmt.Errorf("failed to close writer for topic %s: %w", topic, err)) + } + } + + if len(errs) > 0 { + return fmt.Errorf("errors closing writers: %v", errs) + } + + return nil +} + func (p *Producer) extractMessageKey(env api.Envelope) ([]byte, error) { - if p.cfg.Options.Key.Strategy != "message_id" { + if p.config.Options.Key.Strategy != "message_id" { return nil, nil } diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go index c27433d..7b52810 100644 --- a/transport/kafka/transport.go +++ b/transport/kafka/transport.go @@ -3,23 +3,27 @@ package kafka import ( "context" "fmt" - "log/slog" "net/url" "strings" + "github.com/segmentio/kafka-go" + "github.com/gerfey/messenger/api" ) +type ConnectionKafka interface { + CreateReader(kafka.ReaderConfig) *kafka.Reader + CreateWriter(string, ProducerOptionsConfig, bool, kafka.Balancer) *kafka.Writer +} + type Transport struct { cfg TransportConfig - producer *Producer - consumer *Consumer - serializer api.Serializer - logger *slog.Logger - conn *Connection + producer api.Producer + consumer api.Consumer + connection ConnectionKafka } -func NewTransport(cfg TransportConfig, logger *slog.Logger, ser api.Serializer) (api.Transport, error) { +func NewTransport(cfg TransportConfig, serializer api.Serializer) (api.Transport, error) { u, err := url.Parse(cfg.DSN) if err != nil { return nil, fmt.Errorf("failed to parse dsn: %w", err) @@ -27,27 +31,26 @@ func NewTransport(cfg TransportConfig, logger *slog.Logger, ser api.Serializer) brokers := strings.Split(u.Host, ",") - conn, err := NewConnection(brokers) - if err != nil { - logger.Error("failed to connect to Kafka brokers", "error", err) - - return nil, err + connection, errConnection := NewConnection(brokers) + if errConnection != nil { + return nil, fmt.Errorf("failed to create connection: %w", errConnection) } - producer, err := NewProducer(cfg, ser, conn, logger) - if err != nil { - return nil, fmt.Errorf("failed to create kafka producer: %w", err) + producer, errProducer := NewProducer(cfg, connection, serializer) + if errProducer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errProducer) } - consumer := NewConsumer(cfg, ser, conn, logger) + consumer, errConsumer := NewConsumer(cfg, connection, serializer) + if errConsumer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errConsumer) + } return &Transport{ cfg: cfg, producer: producer, consumer: consumer, - serializer: ser, - logger: logger, - conn: conn, + connection: connection, }, nil } diff --git a/transport/redis/consumer.go b/transport/redis/consumer.go index ca8e270..de9505a 100644 --- a/transport/redis/consumer.go +++ b/transport/redis/consumer.go @@ -3,7 +3,6 @@ package redis import ( "context" "errors" - "log/slog" "strings" "sync" "time" @@ -19,59 +18,55 @@ const ( ) type Consumer struct { - cfg TransportConfig + config TransportConfig serializer api.Serializer - conn *Connection - logger *slog.Logger - ctx context.Context - cancel context.CancelFunc + connection ConnectionRedis wg sync.WaitGroup } -func NewConsumer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) *Consumer { - ctx, cancel := context.WithCancel(context.Background()) - +func NewConsumer(config TransportConfig, serializer api.Serializer, connection ConnectionRedis) (api.Consumer, error) { return &Consumer{ - cfg: cfg, - serializer: ser, - conn: conn, - logger: logger, - ctx: ctx, - cancel: cancel, - } + config: config, + serializer: serializer, + connection: connection, + }, nil } func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - group := c.cfg.Options.Group - stream := c.cfg.Options.Stream + group := c.config.Options.Group + stream := c.config.Options.Stream - _ = c.conn.Client().XGroupCreateMkStream(ctx, stream, group, "$") + _ = c.connection.Client().XGroupCreateMkStream(ctx, stream, group, "$") c.wg.Add(1) go func() { defer c.wg.Done() - c.consumeLoop(handler) + c.consumeLoop(ctx, handler) }() <-ctx.Done() - c.cancel() + c.wg.Wait() return ctx.Err() } -func (c *Consumer) consumeLoop(handler func(context.Context, api.Envelope) error) { - rdb := c.conn.Client() - stream := c.cfg.Options.Stream - group := c.cfg.Options.Group - consumer := c.cfg.Options.Consumer +func (c *Consumer) Close() error { + return nil +} + +func (c *Consumer) consumeLoop(ctx context.Context, handler func(context.Context, api.Envelope) error) { + rdb := c.connection.Client() + stream := c.config.Options.Stream + group := c.config.Options.Group + consumer := c.config.Options.Consumer for { select { - case <-c.ctx.Done(): + case <-ctx.Done(): return default: - streams, err := rdb.XReadGroup(c.ctx, &redis.XReadGroupArgs{ + streams, err := rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: group, Consumer: consumer, Streams: []string{stream, ">"}, @@ -80,32 +75,30 @@ func (c *Consumer) consumeLoop(handler func(context.Context, api.Envelope) error }).Result() if err != nil && !errors.Is(err, redis.Nil) { - c.logger.Error("XREADGROUP error", "error", err) - continue } for _, s := range streams { for _, msg := range s.Messages { - c.handleMessage(msg, handler) + c.handleMessage(ctx, msg, handler) } } } } } -func (c *Consumer) handleMessage(msg redis.XMessage, handler func(context.Context, api.Envelope) error) { +func (c *Consumer) handleMessage( + ctx context.Context, + msg redis.XMessage, + handler func(context.Context, api.Envelope) error, +) { bodyRaw, ok := msg.Values["body"] if !ok { - c.logger.Warn("missing 'body' field in message", "id", msg.ID) - return } bodyBytes, ok := bodyRaw.(string) if !ok { - c.logger.Warn("invalid body format", "id", msg.ID) - return } @@ -120,20 +113,16 @@ func (c *Consumer) handleMessage(msg redis.XMessage, handler func(context.Contex env, errUnmarshal := c.serializer.Unmarshal([]byte(bodyBytes), headers) if errUnmarshal != nil { - c.logger.Error("failed to unmarshal", "error", errUnmarshal) - return } - env = env.WithStamp(stamps.ReceivedStamp{Transport: c.cfg.Name}) - - if errHandler := handler(c.ctx, env); errHandler != nil { - c.logger.Error("handler failed", "error", errHandler) + env = env.WithStamp(stamps.ReceivedStamp{Transport: c.config.Name}) + if errHandler := handler(ctx, env); errHandler != nil { return } - if err := c.conn.Client().XAck(c.ctx, c.cfg.Options.Stream, c.cfg.Options.Group, msg.ID).Err(); err != nil { - c.logger.Error("XACK failed", "id", msg.ID, "error", err) + if err := c.connection.Client().XAck(ctx, c.config.Options.Stream, c.config.Options.Group, msg.ID).Err(); err != nil { + return } } diff --git a/transport/redis/factory.go b/transport/redis/factory.go index 95c390a..c9c456f 100644 --- a/transport/redis/factory.go +++ b/transport/redis/factory.go @@ -2,7 +2,6 @@ package redis import ( "fmt" - "log/slog" "strings" "github.com/creasty/defaults" @@ -11,21 +10,22 @@ import ( "github.com/gerfey/messenger/api" ) -type TransportFactory struct { - logger *slog.Logger -} +type TransportFactory struct{} -func NewTransportFactory(logger *slog.Logger) api.TransportFactory { - return &TransportFactory{ - logger: logger, - } +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} } func (t *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "redis://") } -func (t *TransportFactory) Create(name string, dsn string, options []byte, ser api.Serializer) (api.Transport, error) { +func (t *TransportFactory) Create( + name string, + dsn string, + options []byte, + serializer api.Serializer, +) (api.Transport, error) { var optsConfig OptionsConfig if err := defaults.Set(&optsConfig); err != nil { return nil, fmt.Errorf("set defaults: %w", err) @@ -41,5 +41,5 @@ func (t *TransportFactory) Create(name string, dsn string, options []byte, ser a Options: optsConfig, } - return NewTransport(tCfg, t.logger, ser) + return NewTransport(tCfg, serializer) } diff --git a/transport/redis/producer.go b/transport/redis/producer.go index 11e6125..5440eaf 100644 --- a/transport/redis/producer.go +++ b/transport/redis/producer.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "log/slog" "regexp" "github.com/redis/go-redis/v9" @@ -15,19 +14,17 @@ import ( ) type Producer struct { - cfg TransportConfig + config TransportConfig serializer api.Serializer - conn *Connection - logger *slog.Logger + connection ConnectionRedis } -func NewProducer(cfg TransportConfig, ser api.Serializer, conn *Connection, logger *slog.Logger) *Producer { +func NewProducer(config TransportConfig, serializer api.Serializer, connection ConnectionRedis) (api.Producer, error) { return &Producer{ - cfg: cfg, - serializer: ser, - conn: conn, - logger: logger, - } + config: config, + serializer: serializer, + connection: connection, + }, nil } func (p *Producer) Send(ctx context.Context, env api.Envelope) error { @@ -44,7 +41,7 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { data["header_"+k] = v } - stream := p.cfg.Options.Stream + stream := p.config.Options.Stream if stream == "" { return errors.New("redis: stream name is not configured") } @@ -56,7 +53,7 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { } } - _, err = p.conn.Client().XAdd(ctx, &redis.XAddArgs{ + _, err = p.connection.Client().XAdd(ctx, &redis.XAddArgs{ ID: id, Stream: stream, Values: data, @@ -65,10 +62,10 @@ func (p *Producer) Send(ctx context.Context, env api.Envelope) error { return fmt.Errorf("redis: XADD failed: %w", err) } - p.logger.DebugContext(ctx, "message sent to redis stream", - slog.String("stream", stream), - slog.String("message_type", fmt.Sprintf("%T", env.Message()))) + return nil +} +func (p *Producer) Close() error { return nil } diff --git a/transport/redis/transport.go b/transport/redis/transport.go index 149da75..942f676 100644 --- a/transport/redis/transport.go +++ b/transport/redis/transport.go @@ -3,39 +3,45 @@ package redis import ( "context" "fmt" - "log/slog" "strings" + "github.com/redis/go-redis/v9" + "github.com/gerfey/messenger/api" ) +type ConnectionRedis interface { + Client() *redis.Client +} + type Transport struct { cfg TransportConfig - producer *Producer - consumer *Consumer - serializer api.Serializer - logger *slog.Logger - conn *Connection + producer api.Producer + consumer api.Consumer + connection ConnectionRedis } -func NewTransport(cfg TransportConfig, logger *slog.Logger, ser api.Serializer) (api.Transport, error) { - conn, err := NewConnection(cfg.DSN) - if err != nil { - logger.Error("failed to connect", "error", err) +func NewTransport(cfg TransportConfig, serializer api.Serializer) (api.Transport, error) { + connection, errConnection := NewConnection(cfg.DSN) + if errConnection != nil { + return nil, fmt.Errorf("failed to create connection: %w", errConnection) + } - return nil, err + producer, errProducer := NewProducer(cfg, serializer, connection) + if errProducer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errProducer) } - producer := NewProducer(cfg, ser, conn, logger) - consumer := NewConsumer(cfg, ser, conn, logger) + consumer, errConsumer := NewConsumer(cfg, serializer, connection) + if errConsumer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errConsumer) + } return &Transport{ cfg: cfg, producer: producer, consumer: consumer, - serializer: ser, - logger: logger, - conn: conn, + connection: connection, }, nil } @@ -63,10 +69,14 @@ func (t *Transport) Setup(ctx context.Context) error { stream := t.cfg.Options.Stream group := t.cfg.Options.Group - _, err := t.conn.Client().XGroupCreateMkStream(ctx, stream, group, "$").Result() + _, err := t.connection.Client().XGroupCreateMkStream(ctx, stream, group, "$").Result() if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { return fmt.Errorf("failed to create consumer group: %w", err) } return nil } + +func (t *Transport) Close() error { + return nil +} diff --git a/transport/sync/transport.go b/transport/sync/transport.go index 98994db..0205e23 100644 --- a/transport/sync/transport.go +++ b/transport/sync/transport.go @@ -45,3 +45,7 @@ func (t *Transport) Receive(_ context.Context, _ func(context.Context, api.Envel func (t *Transport) Name() string { return "sync" } + +func (t *Transport) Close() error { + return nil +}