diff --git a/README.md b/README.md index b07e4c2..d25a85b 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,14 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/Gerfey/messenger.svg)](https://pkg.go.dev/github.com/Gerfey/messenger) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -> ⚠️ `v0.7.0` is a pre-release version — feel free to test and report issues! +> `v0.8.0` is a pre-release version — feel free to test and report issues! -> 📚 Full documentation available in the [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) +> Full documentation available in the [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) 🇷🇺 [Русская версия](README.ru.md) -## ✨ Features -- **Multiple Transports**: AMQP (RabbitMQ), In-Memory (sync) +## Features +- **Multiple Transports**: AMQP (RabbitMQ), Kafka, Redis (Stream), In-Memory (sync) - **Middleware Chain**: Extensible middleware system for message processing - **Event-Driven**: Built-in event dispatcher for lifecycle hooks - **Retry Mechanism**: Configurable retry strategies with exponential backoff @@ -22,13 +22,13 @@ - **Stamps System**: Metadata attachment for message tracking - **YAML Configuration**: Easy configuration management with `%env(...)%` support -## 📦 Installation +## Installation > Requires Go 1.24+ ```bash -go get github.com/gerfey/messenger@v0.7.0 +go get github.com/gerfey/messenger@v0.8.0 ``` -## 🚀 Quick Start +## Quick Start ### Define Your Message @@ -83,17 +83,24 @@ bus, _ := m.GetDefaultBus() _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) ``` -## 🔍 More Examples +## More Examples -* ✅ Commands with void return -* ✅ Queries with return value access -* ✅ Retry and Dead Letter Queue -* ✅ Custom Middleware and Transports -* ✅ Event Listeners and Lifecycle Hooks +* Commands with void return +* Queries with return value access +* Retry and Dead Letter Queue +* Custom Middleware and Transports +* Event Listeners and Lifecycle Hooks > See [Usage Scenarios](https://github.com/Gerfey/messenger/wiki/Usage-Scenarios) for commands, queries, return values and advanced use-cases. -## 🤝 Contributing +## Benchmark + +- AMQP (RabbitMQ): [AMQP Transport Benchmark Report](docs/benchmark/AMQP-Benchmark.md) +- Redis (Stream): [Redis Transport Benchmark Report](docs/benchmark/Redis-Benchmark.md) +- Sync: [Sync Transport Benchmark Report](docs/benchmark/Sync-Benchmark.md) +- Kafka (Async): [Kafka Transport Async Benchmark Report](docs/benchmark/Kafka-async-Benchmark.md) + +## Contributing 1. Fork the repository 2. Create your feature branch (`git checkout -b feature/amazing-feature`) @@ -101,15 +108,15 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) 4. Push to the branch (`git push origin feature/amazing-feature`) 5. Open a Pull Request -## ⚖️ License +## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. -## ⭐️ Support +## Support If you find this project useful, please consider starring ⭐️ it and sharing with others! -## 🙏 Acknowledgments +## Acknowledgments - Inspired by [Symfony Messenger](https://symfony.com/doc/current/messenger.html) - Built with ❤️ for the Go community diff --git a/README.ru.md b/README.ru.md index c4c6bf7..859f0f7 100644 --- a/README.ru.md +++ b/README.ru.md @@ -7,16 +7,16 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/Gerfey/messenger.svg)](https://pkg.go.dev/github.com/Gerfey/messenger) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -> ⚠️ Версия `v0.7.0` — это пре-релиз. Тестируйте и сообщайте о багах! +> Версия `v0.8.0` — это пре-релиз. Тестируйте и сообщайте о багах! -> 📚 Полная документация доступна на [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) +> Полная документация доступна на [GitHub Wiki](https://github.com/Gerfey/messenger/wiki/Documentation) 🇬🇧 [English README](README.md) --- -## ✨ Возможности -- **Множественные транспорты**: AMQP (RabbitMQ), In-Memory (`sync`) +## Возможности +- **Множественные транспорты**: AMQP (RabbitMQ), Kafka, Redis (Stream), In-Memory (sync) - **Цепочка middleware**: Расширяемая система промежуточной обработки - **Событийный движок**: Встроенный dispatcher событий жизненного цикла - **Механизм повторов**: Настраиваемые стратегии ретраев с поддержкой DLQ @@ -24,13 +24,13 @@ - **Система метаданных (Stamps)**: Для трассировки и поведения сообщений - **YAML-конфигурация**: С поддержкой переменных окружения `%env(...)%` -## 📦 Установка +## Установка > Требуется Go версии **1.24+** ```bash -go get github.com/gerfey/messenger@v0.7.0 +go get github.com/gerfey/messenger@v0.8.0 ``` -## 🚀 Быстрый старт +## Быстрый старт ### Определите сообщение @@ -85,17 +85,24 @@ bus, _ := m.GetDefaultBus() _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) ``` -## 🔍 Больше примеров +## Больше примеров -* ✅ Команды без возврата значения -* ✅ Запросы с возвратом результата -* ✅ Повторные попытки и Dead Letter Queue -* ✅ Пользовательские middleware и транспорты -* ✅ Слушатели событий и хуки жизненного цикла +* Команды без возврата значения +* Запросы с возвратом результата +* Повторные попытки и Dead Letter Queue +* Пользовательские middleware и транспорты +* Слушатели событий и хуки жизненного цикла > Смотри [Сценарии использования](https://github.com/Gerfey/messenger/wiki/Сценарии-использования). -## 🤝 Как внести вклад +## Показатели + +- AMQP (RabbitMQ): [AMQP Transport Benchmark Report](docs/benchmark/AMQP-Benchmark.md) +- Redis (Stream): [Redis Transport Benchmark Report](docs/benchmark/Redis-Benchmark.md) +- Sync: [Sync Transport Benchmark Report](docs/benchmark/Sync-Benchmark.md) +- Kafka (Async): [Kafka Transport Async Benchmark Report](docs/benchmark/Kafka-async-Benchmark.md) + +## Как внести вклад 1. Форкните репозиторий 2. Создайте новую ветку (`git checkout -b feature/amazing-feature`) @@ -103,15 +110,15 @@ _, _ = bus.Dispatch(ctx, &HelloMessage{Text: "World"}) 4. Запушьте изменения (`git push origin feature/amazing-feature`) 5. Откройте Pull Request -## ⚖️ Лицензия +## Лицензия Проект лицензирован под [MIT](LICENSE). -## ⭐️ Поддержка +## Поддержка Если вам полезен этот проект — поставьте ⭐️ и расскажите другим! -## 🙏 Благодарности +## Благодарности - Вдохновлено [Symfony Messenger](https://symfony.com/doc/current/messenger.html) - Сделано с ❤️ для сообщества Go diff --git a/api/builder.go b/api/builder.go index de18d3a..8dcdc98 100644 --- a/api/builder.go +++ b/api/builder.go @@ -6,6 +6,7 @@ type Builder interface { RegisterStamp(any) RegisterListener(any, any) RegisterMiddleware(string, Middleware) + RegisterSerializer(string, Serializer) RegisterTransportFactory(TransportFactory) Build() (Messenger, error) } diff --git a/api/serializer.go b/api/serializer.go index 1ccf26c..74d16b4 100644 --- a/api/serializer.go +++ b/api/serializer.go @@ -4,3 +4,9 @@ type Serializer interface { Marshal(Envelope) ([]byte, map[string]string, error) Unmarshal(body []byte, headers map[string]string) (Envelope, error) } + +type SerializerLocator interface { + Register(string, Serializer) + GetAll() []Serializer + Get(name string) (Serializer, error) +} diff --git a/api/transport.go b/api/transport.go index 7087681..242acde 100644 --- a/api/transport.go +++ b/api/transport.go @@ -3,13 +3,12 @@ package api import ( "context" "reflect" - - "github.com/gerfey/messenger/config" ) type Transport interface { Sender Receiver + Closer } type Sender interface { @@ -21,11 +20,30 @@ type Receiver interface { Receive(context.Context, func(context.Context, Envelope) error) error } +type Closer interface { + Close() error +} + +type Producer interface { + Send(context.Context, Envelope) error + Close() error +} + +type Consumer interface { + Consume(context.Context, func(context.Context, Envelope) error) error + Close() error +} + type RetryableTransport interface { Transport Retry(context.Context, Envelope) error } +type SetupableTransport interface { + Transport + Setup(ctx context.Context) error +} + type SenderLocator interface { Register(string, Sender) error GetSenders(Envelope) []Sender @@ -35,7 +53,7 @@ type SenderLocator interface { type TransportFactory interface { Supports(string) bool - Create(string, string, config.OptionsConfig) (Transport, error) + Create(string, string, []byte, Serializer) (Transport, error) } type RoutedMessage interface { diff --git a/builder/builder.go b/core/builder/builder.go similarity index 84% rename from builder/builder.go rename to core/builder/builder.go index 8930ca1..e80e761 100644 --- a/builder/builder.go +++ b/core/builder/builder.go @@ -8,8 +8,8 @@ import ( "github.com/gerfey/messenger" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/core/bus" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/core/event" "github.com/gerfey/messenger/core/handler" "github.com/gerfey/messenger/core/listener" @@ -17,10 +17,13 @@ import ( "github.com/gerfey/messenger/core/middleware/implementation" "github.com/gerfey/messenger/core/retry" "github.com/gerfey/messenger/core/routing" + "github.com/gerfey/messenger/core/serializer" "github.com/gerfey/messenger/core/stamps" "github.com/gerfey/messenger/transport" "github.com/gerfey/messenger/transport/amqp" "github.com/gerfey/messenger/transport/inmemory" + "github.com/gerfey/messenger/transport/kafka" + "github.com/gerfey/messenger/transport/redis" "github.com/gerfey/messenger/transport/sync" ) @@ -31,6 +34,7 @@ type Builder struct { handlersLocator api.HandlerLocator senderLocator api.SenderLocator middlewareLocator api.MiddlewareLocator + serializerLocator api.SerializerLocator busLocator api.BusLocator eventDispatcher api.EventDispatcher logger *slog.Logger @@ -38,22 +42,25 @@ type Builder struct { func NewBuilder(cfg *config.MessengerConfig, logger *slog.Logger) api.Builder { resolver := NewResolver() - busLocator := bus.NewLocator() - - tf := transport.NewFactoryChain( - amqp.NewTransportFactory(logger, resolver), - inmemory.NewTransportFactory(logger, resolver), - sync.NewTransportFactory(logger, busLocator), + serializerLocator := serializer.NewSerializerLocator() + + transportFactory := transport.NewFactoryChain( + sync.NewTransportFactory(busLocator), + inmemory.NewTransportFactory(), + amqp.NewTransportFactory(), + kafka.NewTransportFactory(), + redis.NewTransportFactory(), ) return &Builder{ cfg: cfg, resolver: resolver, - transportFactory: tf, + transportFactory: transportFactory, handlersLocator: handler.NewHandlerLocator(), senderLocator: transport.NewSenderLocator(), middlewareLocator: middleware.NewMiddlewareLocator(), + serializerLocator: serializerLocator, busLocator: busLocator, eventDispatcher: event.NewEventDispatcher(logger), logger: logger, @@ -69,13 +76,13 @@ func (b *Builder) RegisterHandler(handler any) error { return fmt.Errorf("register handler: %w", err) } - for _, h := range b.handlersLocator.GetAll() { - b.resolver.Register(h.InputType.String(), h.InputType) - } - return nil } +func (b *Builder) RegisterSerializer(name string, sz api.Serializer) { + b.serializerLocator.Register(name, sz) +} + func (b *Builder) RegisterMiddleware(name string, mw api.Middleware) { b.middlewareLocator.Register(name, mw) } @@ -95,8 +102,14 @@ func (b *Builder) RegisterListener(event any, listener any) { } func (b *Builder) Build() (api.Messenger, error) { + for _, h := range b.handlersLocator.GetAll() { + b.resolver.Register(h.InputType.String(), h.InputType) + } + b.registerStamps() + b.serializerLocator.Register("default.transport.serializer", serializer.NewSerializer(b.resolver)) + if err := b.setupBuses(); err != nil { return nil, err } @@ -117,6 +130,8 @@ func (b *Builder) setupBuses() error { } chain = append(chain, implementation.NewAddBusNameMiddleware(name)) + chain = append(chain, implementation.NewAddMessageIDMiddleware()) + chain = append( chain, implementation.NewSendMessageMiddleware(b.logger, b.senderLocator, b.eventDispatcher), @@ -164,7 +179,7 @@ func (b *Builder) setupRouting() (api.Router, error) { router := routing.NewRouter() for msgTypeStr, transportName := range b.cfg.Routing { - t, err := b.handlersLocator.ResolveMessageType(msgTypeStr) + t, err := b.resolver.ResolveMessageType(msgTypeStr) if err != nil { return nil, fmt.Errorf("failed to resolve message type '%s' in routing configuration: %w", msgTypeStr, err) } @@ -214,7 +229,17 @@ func (b *Builder) createTransports(manager *transport.Manager) (map[string]api.T b.createdSyncTransport(createdTransports) for name, tCfg := range b.cfg.Transports { - tr, err := b.transportFactory.CreateTransport(name, tCfg) + serializerName := tCfg.Serializer + if serializerName == "" { + serializerName = b.cfg.DefaultSerializer + } + + sz, err := b.serializerLocator.Get(serializerName) + if err != nil { + return nil, nil, fmt.Errorf("serializer %q not found for transport %q: %w", serializerName, name, err) + } + + tr, err := b.transportFactory.CreateTransport(name, tCfg, sz) if err != nil { return nil, nil, fmt.Errorf("failed to create transport '%s': %w", name, err) } @@ -269,15 +294,15 @@ func (b *Builder) setupRetryListeners(createdTransports map[string]api.Transport func (b *Builder) registerStamps() { b.resolver.RegisterStamp(stamps.BusNameStamp{}) b.resolver.RegisterStamp(stamps.RedeliveryStamp{}) + b.resolver.RegisterStamp(stamps.MessageIDStamp{}) } func (b *Builder) createdSyncTransport(createdTransports map[string]api.Transport) { cfg := config.TransportConfig{ - DSN: "sync://", - Options: config.OptionsConfig{}, + DSN: "sync://", } - if syncTransport, err := b.transportFactory.CreateTransport("sync", cfg); err == nil { + if syncTransport, err := b.transportFactory.CreateTransport("sync", cfg, nil); err == nil { createdTransports["sync"] = syncTransport } } diff --git a/builder/builder_test.go b/core/builder/builder_test.go similarity index 91% rename from builder/builder_test.go rename to core/builder/builder_test.go index 57913d0..23d0ea3 100644 --- a/builder/builder_test.go +++ b/core/builder/builder_test.go @@ -7,8 +7,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/core/stamps" "github.com/gerfey/messenger/tests/helpers" ) @@ -25,9 +26,9 @@ func TestNewBuilder(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -201,7 +202,8 @@ func TestBuilder_RegisterListener(t *testing.T) { func TestBuilder_RegisterTransportFactory(t *testing.T) { t.Run("register transport factory", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, @@ -210,9 +212,9 @@ func TestBuilder_RegisterTransportFactory(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -256,7 +258,8 @@ func TestBuilder_RegisterTransportFactory(t *testing.T) { func TestBuilder_Build_Errors(t *testing.T) { t.Run("build fails with unknown message type in routing", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, @@ -265,9 +268,9 @@ func TestBuilder_Build_Errors(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -296,9 +299,9 @@ func TestBuilder_Build_Errors(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -319,7 +322,8 @@ func TestBuilder_Build_Errors(t *testing.T) { func TestBuilder_Build_Success(t *testing.T) { t.Run("build messenger with complete configuration", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{"test_middleware"}, @@ -331,9 +335,9 @@ func TestBuilder_Build_Success(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -368,7 +372,8 @@ func TestBuilder_Build_Success(t *testing.T) { t.Run("build messenger with retry configuration", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, @@ -377,9 +382,9 @@ func TestBuilder_Build_Success(t *testing.T) { Transports: map[string]config.TransportConfig{ "inmemory": { DSN: "in-memory://test", - Options: config.OptionsConfig{ - AutoSetup: true, - ConsumerPoolSize: 10, + Options: map[string]any{ + "auto_setup": true, + "consumer_pool_size": 10, }, }, }, @@ -404,7 +409,8 @@ func TestBuilder_Build_Success(t *testing.T) { t.Run("build messenger with multiple buses and transports", func(t *testing.T) { cfg := &config.MessengerConfig{ - DefaultBus: "default", + DefaultBus: "default", + DefaultSerializer: "default.transport.serializer", Buses: map[string]config.BusConfig{ "default": { Middleware: []string{}, diff --git a/builder/resolver.go b/core/builder/resolver.go similarity index 100% rename from builder/resolver.go rename to core/builder/resolver.go diff --git a/builder/resolver_test.go b/core/builder/resolver_test.go similarity index 99% rename from builder/resolver_test.go rename to core/builder/resolver_test.go index ea6686a..e7e7b42 100644 --- a/builder/resolver_test.go +++ b/core/builder/resolver_test.go @@ -7,7 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/stamps" "github.com/gerfey/messenger/tests/helpers" ) diff --git a/config/config.go b/core/config/config.go similarity index 54% rename from config/config.go rename to core/config/config.go index 7ad9b63..9f4d879 100644 --- a/config/config.go +++ b/core/config/config.go @@ -11,11 +11,12 @@ const ( ) type MessengerConfig struct { - DefaultBus string `yaml:"default_bus" default:"default"` - FailureTransport string `yaml:"failure_transport"` - Buses map[string]BusConfig `yaml:"buses"` - Transports map[string]TransportConfig `yaml:"transports"` - Routing map[string]string `yaml:"routing"` + DefaultBus string `yaml:"default_bus" default:"default"` + DefaultSerializer string `yaml:"default_serializer" default:"default.transport.serializer"` + FailureTransport string `yaml:"failure_transport"` + Buses map[string]BusConfig `yaml:"buses"` + Transports map[string]TransportConfig `yaml:"transports"` + Routing map[string]string `yaml:"routing"` } type BusConfig struct { @@ -24,8 +25,9 @@ type BusConfig struct { type TransportConfig struct { DSN string `yaml:"dsn"` + Serializer string `yaml:"serializer"` RetryStrategy *RetryStrategyConfig `yaml:"retry_strategy"` - Options OptionsConfig `yaml:"options"` + Options map[string]any `yaml:"options"` } type RetryStrategyConfig struct { @@ -35,28 +37,6 @@ type RetryStrategyConfig struct { MaxDelay time.Duration `yaml:"max_delay"` } -type OptionsConfig struct { - AutoSetup bool `yaml:"auto_setup" default:"true"` - ConsumerPoolSize int `yaml:"consumer_pool_size" default:"10"` - Exchange ExchangeConfig `yaml:"exchange"` - Queues map[string]Queue `yaml:"queues"` -} - -type ExchangeConfig struct { - Name string `yaml:"name"` - Type string `yaml:"type" default:"topic"` // topic, direct, fanout - Durable bool `yaml:"durable" default:"true"` - AutoDelete bool `yaml:"auto_delete" default:"false"` - Internal bool `yaml:"internal" default:"false"` -} - -type Queue struct { - BindingKeys []string `yaml:"binding_keys"` - Durable bool `yaml:"durable" default:"true"` - Exclusive bool `yaml:"exclusive" default:"false"` - AutoDelete bool `yaml:"auto_delete" default:"false"` -} - type SerializedEnvelope struct { Message any `json:"message"` MessageType string `json:"type"` diff --git a/config/config_test.go b/core/config/config_test.go similarity index 63% rename from config/config_test.go rename to core/config/config_test.go index 9d8532a..3d1c436 100644 --- a/config/config_test.go +++ b/core/config/config_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) type testProcessor struct { @@ -25,7 +25,7 @@ func (p *testProcessor) Process(content []byte) ([]byte, error) { func TestLoadConfig(t *testing.T) { t.Run("load valid config", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" cfg, err := config.LoadConfig(path) require.NoError(t, err) @@ -37,7 +37,7 @@ func TestLoadConfig(t *testing.T) { }) t.Run("load config with env variables", func(t *testing.T) { - path := "../tests/fixtures/configs/config_with_env.yaml" + path := "../../tests/fixtures/configs/config_with_env.yaml" t.Setenv("MEMORY_HOST", "test-host") t.Setenv("CONSUMER_POOL_SIZE", "15") @@ -54,7 +54,7 @@ func TestLoadConfig(t *testing.T) { require.NoError(t, err) assert.Equal(t, "memory://test-host", cfg.Transports["default"].DSN) - assert.Equal(t, 15, cfg.Transports["default"].Options.ConsumerPoolSize) + assert.Equal(t, 15, cfg.Transports["default"].Options["consumer_pool_size"]) assert.Equal(t, "amqp://guest:guest@localhost:5672/", cfg.Transports["amqp"].DSN) assert.Equal(t, uint(5), cfg.Transports["amqp"].RetryStrategy.MaxRetries) }) @@ -68,7 +68,7 @@ func TestLoadConfig(t *testing.T) { }) t.Run("load invalid config", func(t *testing.T) { - path := "../tests/fixtures/configs/invalid_config.yaml" + path := "../../tests/fixtures/configs/invalid_config.yaml" cfg, err := config.LoadConfig(path) require.Error(t, err) @@ -76,7 +76,7 @@ func TestLoadConfig(t *testing.T) { }) t.Run("load with custom processor", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" customProcessor := &testProcessor{ processFunc: func(content []byte) ([]byte, error) { @@ -121,8 +121,14 @@ transports: require.NoError(t, err) assert.Equal(t, "memory://default", cfg.Transports["default"].DSN) - assert.True(t, cfg.Transports["default"].Options.AutoSetup) - assert.Equal(t, 10, cfg.Transports["default"].Options.ConsumerPoolSize) + autoSetup, ok := cfg.Transports["default"].Options["auto_setup"] + if ok && autoSetup != nil { + assert.True(t, autoSetup.(bool)) + } + consumerPoolSize, ok := cfg.Transports["default"].Options["consumer_pool_size"] + if ok && consumerPoolSize != nil { + assert.Equal(t, 10, consumerPoolSize) + } }) t.Run("exchange options default values", func(t *testing.T) { @@ -139,11 +145,33 @@ transports: `), &cfg) require.NoError(t, err) - assert.Equal(t, "messages", cfg.Transports["amqp"].Options.Exchange.Name) - assert.Equal(t, "topic", cfg.Transports["amqp"].Options.Exchange.Type) - assert.True(t, cfg.Transports["amqp"].Options.Exchange.Durable) - assert.False(t, cfg.Transports["amqp"].Options.Exchange.AutoDelete) - assert.False(t, cfg.Transports["amqp"].Options.Exchange.Internal) + exchangeOptions, ok := cfg.Transports["amqp"].Options["exchange"].(map[string]any) + require.True(t, ok) + + name, ok := exchangeOptions["name"] + if ok && name != nil { + assert.Equal(t, "messages", name) + } + + exchangeType, ok := exchangeOptions["type"] + if ok && exchangeType != nil { + assert.Equal(t, "topic", exchangeType) + } + + durable, ok := exchangeOptions["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } + + autoDelete, ok := exchangeOptions["auto_delete"] + if ok && autoDelete != nil { + assert.False(t, autoDelete.(bool)) + } + + internal, ok := exchangeOptions["internal"] + if ok && internal != nil { + assert.False(t, internal.(bool)) + } }) t.Run("queue options default values", func(t *testing.T) { @@ -162,9 +190,31 @@ transports: `), &cfg) require.NoError(t, err) - assert.Contains(t, cfg.Transports["amqp"].Options.Queues["default"].BindingKeys, "#") - assert.True(t, cfg.Transports["amqp"].Options.Queues["default"].Durable) - assert.False(t, cfg.Transports["amqp"].Options.Queues["default"].Exclusive) - assert.False(t, cfg.Transports["amqp"].Options.Queues["default"].AutoDelete) + + queuesOptions, ok := cfg.Transports["amqp"].Options["queues"].(map[string]any) + require.True(t, ok) + + defaultQueue, ok := queuesOptions["default"].(map[string]any) + require.True(t, ok) + + bindingKeys, ok := defaultQueue["binding_keys"] + if ok && bindingKeys != nil { + assert.Contains(t, bindingKeys, "#") + } + + durable, ok := defaultQueue["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } + + exclusive, ok := defaultQueue["exclusive"] + if ok && exclusive != nil { + assert.False(t, exclusive.(bool)) + } + + autoDelete, ok := defaultQueue["auto_delete"] + if ok && autoDelete != nil { + assert.False(t, autoDelete.(bool)) + } }) } diff --git a/config/env_processor.go b/core/config/env_processor.go similarity index 100% rename from config/env_processor.go rename to core/config/env_processor.go diff --git a/config/env_processor_test.go b/core/config/env_processor_test.go similarity index 99% rename from config/env_processor_test.go rename to core/config/env_processor_test.go index b166454..4589de8 100644 --- a/config/env_processor_test.go +++ b/core/config/env_processor_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) func TestEnvVarProcessor_Process(t *testing.T) { diff --git a/config/parser.go b/core/config/parser.go similarity index 100% rename from config/parser.go rename to core/config/parser.go diff --git a/config/parser_test.go b/core/config/parser_test.go similarity index 67% rename from config/parser_test.go rename to core/config/parser_test.go index 6d72117..3a84814 100644 --- a/config/parser_test.go +++ b/core/config/parser_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) func TestYAMLParser_Parse(t *testing.T) { @@ -42,8 +42,16 @@ transports: require.NoError(t, err) assert.Equal(t, "custom", cfg.DefaultBus) - assert.True(t, cfg.Transports["default"].Options.AutoSetup) - assert.Equal(t, 10, cfg.Transports["default"].Options.ConsumerPoolSize) + + autoSetup, ok := cfg.Transports["default"].Options["auto_setup"] + if ok && autoSetup != nil { + assert.True(t, autoSetup.(bool)) + } + + consumerPoolSize, ok := cfg.Transports["default"].Options["consumer_pool_size"] + if ok && consumerPoolSize != nil { + assert.Equal(t, 10, consumerPoolSize) + } }) t.Run("parse invalid yaml", func(t *testing.T) { @@ -86,10 +94,29 @@ transports: err := parser.Parse(content, &cfg) require.NoError(t, err) - assert.Equal(t, "messages", cfg.Transports["amqp"].Options.Exchange.Name) - assert.Equal(t, "topic", cfg.Transports["amqp"].Options.Exchange.Type) - assert.True(t, cfg.Transports["amqp"].Options.Exchange.Durable) - assert.False(t, cfg.Transports["amqp"].Options.Exchange.AutoDelete) + + exchangeOptions, ok := cfg.Transports["amqp"].Options["exchange"].(map[string]interface{}) + require.True(t, ok) + + name, ok := exchangeOptions["name"] + if ok && name != nil { + assert.Equal(t, "messages", name) + } + + exchangeType, ok := exchangeOptions["type"] + if ok && exchangeType != nil { + assert.Equal(t, "topic", exchangeType) + } + + durable, ok := exchangeOptions["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } + + autoDelete, ok := exchangeOptions["auto_delete"] + if ok && autoDelete != nil { + assert.False(t, autoDelete.(bool)) + } }) t.Run("parse yaml with queue options", func(t *testing.T) { @@ -107,9 +134,23 @@ transports: err := parser.Parse(content, &cfg) require.NoError(t, err) - assert.Contains(t, cfg.Transports["amqp"].Options.Queues, "default") - assert.Contains(t, cfg.Transports["amqp"].Options.Queues["default"].BindingKeys, "#") - assert.True(t, cfg.Transports["amqp"].Options.Queues["default"].Durable) + + queuesOptions, ok := cfg.Transports["amqp"].Options["queues"].(map[string]interface{}) + require.True(t, ok) + assert.Contains(t, queuesOptions, "default") + + defaultQueue, ok := queuesOptions["default"].(map[string]interface{}) + require.True(t, ok) + + bindingKeys, ok := defaultQueue["binding_keys"].([]interface{}) + if ok && bindingKeys != nil { + assert.Contains(t, bindingKeys, "#") + } + + durable, ok := defaultQueue["durable"] + if ok && durable != nil { + assert.True(t, durable.(bool)) + } }) t.Run("parse yaml with retry strategy", func(t *testing.T) { @@ -138,7 +179,7 @@ func TestYAMLParser_Integration(t *testing.T) { reader := &config.FileReader{} t.Run("parse real config file", func(t *testing.T) { - content, err := reader.Read("../tests/fixtures/configs/valid_config.yaml") + content, err := reader.Read("../../tests/fixtures/configs/valid_config.yaml") require.NoError(t, err) var cfg config.MessengerConfig diff --git a/config/reader.go b/core/config/reader.go similarity index 100% rename from config/reader.go rename to core/config/reader.go diff --git a/config/reader_test.go b/core/config/reader_test.go similarity index 89% rename from config/reader_test.go rename to core/config/reader_test.go index 5dff18a..454306a 100644 --- a/config/reader_test.go +++ b/core/config/reader_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" ) type MockProcessor struct { @@ -27,7 +27,7 @@ func TestFileReader_Read(t *testing.T) { reader := &config.FileReader{} t.Run("read existing file", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" content, err := reader.Read(path) require.NoError(t, err) @@ -45,7 +45,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with single processor", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" processor := &MockProcessor{ ProcessFunc: func(content []byte) ([]byte, error) { return append(content, []byte("\n# Processed")...), nil @@ -59,7 +59,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with multiple processors", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" processor1 := &MockProcessor{ ProcessFunc: func(content []byte) ([]byte, error) { return append(content, []byte("\n# Processor1")...), nil @@ -79,7 +79,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with processor error", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" expectedErr := errors.New("processor error") processor := &MockProcessor{ ProcessFunc: func(_ []byte) ([]byte, error) { @@ -95,7 +95,7 @@ func TestFileReader_Read(t *testing.T) { }) t.Run("read with multiple processors, second fails", func(t *testing.T) { - path := "../tests/fixtures/configs/valid_config.yaml" + path := "../../tests/fixtures/configs/valid_config.yaml" expectedErr := errors.New("processor 2 error") processor1 := &MockProcessor{ ProcessFunc: func(content []byte) ([]byte, error) { @@ -135,7 +135,7 @@ func TestFileReader_Integration(t *testing.T) { reader := &config.FileReader{} t.Run("read config with env processor", func(t *testing.T) { - path := "../tests/fixtures/configs/config_with_env.yaml" + path := "../../tests/fixtures/configs/config_with_env.yaml" t.Setenv("MEMORY_HOST", "localhost") t.Setenv("CONSUMER_POOL_SIZE", "20") @@ -150,7 +150,7 @@ func TestFileReader_Integration(t *testing.T) { }) t.Run("read config with multiple processors", func(t *testing.T) { - path := "../tests/fixtures/configs/config_with_env.yaml" + path := "../../tests/fixtures/configs/config_with_env.yaml" t.Setenv("MEMORY_HOST", "localhost") diff --git a/core/middleware/implementation/add_message_id.go b/core/middleware/implementation/add_message_id.go new file mode 100644 index 0000000..76e2b68 --- /dev/null +++ b/core/middleware/implementation/add_message_id.go @@ -0,0 +1,32 @@ +package implementation + +import ( + "context" + + "github.com/google/uuid" + + "github.com/gerfey/messenger/core/envelope" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" +) + +type AddMessageIDMiddleware struct{} + +func NewAddMessageIDMiddleware() *AddMessageIDMiddleware { + return &AddMessageIDMiddleware{} +} + +func (m *AddMessageIDMiddleware) Handle( + ctx context.Context, + env api.Envelope, + next api.NextFunc, +) (api.Envelope, error) { + if _, ok := envelope.LastStampOf[stamps.MessageIDStamp](env); !ok { + env = env.WithStamp(stamps.MessageIDStamp{ + MessageID: uuid.New().String(), + }) + } + + return next(ctx, env) +} diff --git a/core/serializer/locator.go b/core/serializer/locator.go new file mode 100644 index 0000000..46605bb --- /dev/null +++ b/core/serializer/locator.go @@ -0,0 +1,39 @@ +package serializer + +import ( + "fmt" + + "github.com/gerfey/messenger/api" +) + +type Locator struct { + serializers map[string]api.Serializer +} + +func NewSerializerLocator() api.SerializerLocator { + return &Locator{ + serializers: make(map[string]api.Serializer), + } +} + +func (m *Locator) Register(name string, serializer api.Serializer) { + m.serializers[name] = serializer +} + +func (m *Locator) GetAll() []api.Serializer { + all := make([]api.Serializer, 0) + for _, serializer := range m.serializers { + all = append(all, serializer) + } + + return all +} + +func (m *Locator) Get(name string) (api.Serializer, error) { + mw, ok := m.serializers[name] + if !ok { + return nil, fmt.Errorf("no serializer with name %s found", name) + } + + return mw, nil +} diff --git a/serializer/serializer.go b/core/serializer/serializer.go similarity index 98% rename from serializer/serializer.go rename to core/serializer/serializer.go index 7ae0dae..c22501d 100644 --- a/serializer/serializer.go +++ b/core/serializer/serializer.go @@ -6,7 +6,7 @@ import ( "reflect" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/core/envelope" ) diff --git a/serializer/serializer_test.go b/core/serializer/serializer_test.go similarity index 98% rename from serializer/serializer_test.go rename to core/serializer/serializer_test.go index 681d956..2352789 100644 --- a/serializer/serializer_test.go +++ b/core/serializer/serializer_test.go @@ -6,9 +6,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gerfey/messenger/builder" + "github.com/gerfey/messenger/core/builder" + + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/core/envelope" - "github.com/gerfey/messenger/serializer" "github.com/gerfey/messenger/tests/helpers" ) diff --git a/core/stamps/message_stamps.go b/core/stamps/message_stamps.go new file mode 100644 index 0000000..7412aa0 --- /dev/null +++ b/core/stamps/message_stamps.go @@ -0,0 +1,5 @@ +package stamps + +type MessageIDStamp struct { + MessageID string +} diff --git a/docs/benchmark/AMQP-Benchmark.md b/docs/benchmark/AMQP-Benchmark.md new file mode 100644 index 0000000..c5a7056 --- /dev/null +++ b/docs/benchmark/AMQP-Benchmark.md @@ -0,0 +1,26 @@ +# AMQP Transport Benchmark Report + +* Transport: `amqp://` (Messenger `v0.8.0`) +* Publishing mode: sync + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------------------- | -----------: | -------------------: | ------------: | --------: | +| `BenchmarkSend` | 378,376 | \~2,643 | 6,792 | 130 | +| `BenchmarkConcurrentSend` | 245,631 | \~4,071 | 6,790 | 129 | + +*Parallel sending gives ~1.5x increase in bandwidth with similar memory consumption.* + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------ | -----------: | -------------------: | ------------: | --------: | +| 100 B | 384,571 | \~2,600 | 6,797 | 130 | +| 1 KB | 397,139 | \~2,518 | 9,058 | 130 | +| 10 KB | 445,552 | \~2,244 | 31,667 | 130 | +| 100 KB | 1,397,238 | \~716 | 320,669 | 136 | + +*Increasing the payload size is expected to reduce RPS and increase memory consumption.* diff --git a/docs/benchmark/Kafka-async-Benchmark.md b/docs/benchmark/Kafka-async-Benchmark.md new file mode 100644 index 0000000..273781e --- /dev/null +++ b/docs/benchmark/Kafka-async-Benchmark.md @@ -0,0 +1,28 @@ +# Kafka Transport Async Benchmark Report + +* Transport: `kafka://` (Messenger `v0.8.0`) +* Publishing mode: async (`async=true`) + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------------------- | -----------: | -------------------: | ------------: | --------: | +| `BenchmarkSend` | 3,018 | \~331,345 | 3,072 | 47 | +| `BenchmarkConcurrentSend` | 1,859 | \~537,924 | 3,067 | 47 | + + +*Async publishing provides high throughput with a stable allocation profile.* + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------ | -----------: | -------------------: | ------------: | --------: | +| 100 B | 2,913 | \~343,289 | 3,062 | 47 | +| 1 KB | 4,147 | \~241,138 | 5,304 | 47 | +| 10 KB | 9,800 | \~102,041 | 27,723 | 47 | +| 100 KB | 87,745 | \~11,397 | 254,397 | 48 | + + +*Increasing the message size is expected to reduce throughput and increase memory consumption, but the curve looks smooth and predictable.* diff --git a/docs/benchmark/Redis-Benchmark.md b/docs/benchmark/Redis-Benchmark.md new file mode 100644 index 0000000..d3d3e69 --- /dev/null +++ b/docs/benchmark/Redis-Benchmark.md @@ -0,0 +1,24 @@ +# Redis Transport Benchmark Report + +* Transport: `redis://` (Messenger `v0.8.0`) +* Publishing mode: sync + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/s) | Memory (B/op) | Allocs/op | +| ------------------------- | ------------ | ------------------ | ------------- | --------- | +| `BenchmarkSend` | 133,748 | \~7,477 | 7,361 | 106 | +| `BenchmarkConcurrentSend` | 32,616 | \~30,675 | 7,397 | 106 | + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/s) | Memory (B/op) | Allocs/op | +| ------------ | ------------ | ------------------ | ------------- | --------- | +| 100 B | 127,118 | \~7,866 | 7,362 | 106 | +| 1 KB | 139,020 | \~7,193 | 9,619 | 106 | +| 10 KB | 219,779 | \~4,551 | 32,072 | 106 | +| 100 KB | 1,009,267 | \~991 | 274,182 | 107 | + +*Redis Streams shows good rps in parallel, but as the payload increases, the speed decreases and memory increases.* diff --git a/docs/benchmark/Sync-Benchmark.md b/docs/benchmark/Sync-Benchmark.md new file mode 100644 index 0000000..0675440 --- /dev/null +++ b/docs/benchmark/Sync-Benchmark.md @@ -0,0 +1,24 @@ +# Sync Transport Benchmark Report + +* Transport: `sync://` (Messenger `v0.8.0`) +* Publishing mode: sync + +## Overall Performance + +| Benchmark | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------------------- | -----------: | -------------------: | ------------: | --------: | +| `BenchmarkSend` | 1,110 | \~900,901 | 1,368 | 38 | +| `BenchmarkConcurrentSend` | 932.9 | \~1,071,926 | 1,377 | 38 | + +--- + +## Message Size Impact + +| Message Size | Time (ns/op) | Throughput (msg/sec) | Memory (B/op) | Allocs/op | +| ------------ | -----------: | -------------------: | ------------: | --------: | +| 100 B | 1,152 | \~868,056 | 1,368 | 38 | +| 1 KB | 1,250 | \~800,000 | 2,281 | 38 | +| 10 KB | 2,091 | \~478,240 | 11,505 | 38 | +| 100 KB | 11,583 | \~86,333 | 107,832 | 38 | + +*ultra‑low latency and allocations — ideal for unit tests, CQRS commands/queries and inline middleware.* diff --git a/examples/config/config.go b/examples/config/config.go index 96e4137..f402025 100644 --- a/examples/config/config.go +++ b/examples/config/config.go @@ -6,7 +6,11 @@ import ( "log/slog" "os" - "github.com/gerfey/messenger/config" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/core/config" + + "github.com/gerfey/messenger/transport/amqp" ) func main() { @@ -39,13 +43,18 @@ func main() { fmt.Printf(" MaxDelay: %v\n", transport.RetryStrategy.MaxDelay) } + rawYAML, _ := yaml.Marshal(transport.Options) + + var opts amqp.OptionsConfig + _ = yaml.Unmarshal(rawYAML, &opts) + fmt.Printf(" Options:\n") - fmt.Printf(" AutoSetup: %v\n", transport.Options.AutoSetup) + fmt.Printf(" AutoSetup: %v\n", opts.AutoSetup) fmt.Printf(" Exchange:\n") - fmt.Printf(" Name: %s\n", transport.Options.Exchange.Name) - fmt.Printf(" Type: %s\n", transport.Options.Exchange.Type) - fmt.Printf(" Durable: %v\n", transport.Options.Exchange.Durable) - fmt.Printf(" AutoDelete: %v\n", transport.Options.Exchange.AutoDelete) - fmt.Printf(" Internal: %v\n", transport.Options.Exchange.Internal) + fmt.Printf(" Name: %s\n", opts.Exchange.Name) + fmt.Printf(" Type: %s\n", opts.Exchange.Type) + fmt.Printf(" Durable: %v\n", opts.Exchange.Durable) + fmt.Printf(" AutoDelete: %v\n", opts.Exchange.AutoDelete) + fmt.Printf(" Internal: %v\n", opts.Exchange.Internal) } } diff --git a/examples/kafka_transport/handler/hello_handler.go b/examples/kafka_transport/handler/hello_handler.go new file mode 100644 index 0000000..5488524 --- /dev/null +++ b/examples/kafka_transport/handler/hello_handler.go @@ -0,0 +1,20 @@ +package handler + +import ( + "context" + "fmt" + + "github.com/gerfey/messenger/examples/kafka_transport/message" +) + +type ExampleHelloHandler struct{} + +func (u *ExampleHelloHandler) Handle(_ context.Context, msg *message.ExampleHelloMessage) error { + fmt.Printf("Handled: Text=%v\n", msg.Text) + + return nil +} + +func (u *ExampleHelloHandler) GetBusName() string { + return "default" +} diff --git a/examples/kafka_transport/kafka_transport.go b/examples/kafka_transport/kafka_transport.go new file mode 100644 index 0000000..95ab734 --- /dev/null +++ b/examples/kafka_transport/kafka_transport.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/examples/kafka_transport/handler" + "github.com/gerfey/messenger/examples/kafka_transport/message" +) + +const ( + waitDurationSeconds = 20 +) + +func main() { + ctx := context.Background() + + log := slog.Default() + + cfg, err := config.LoadConfig("./examples/kafka_transport/messenger.yaml") + if err != nil { + log.Error("ERROR load config", "error", err) + + return + } + + b := builder.NewBuilder(cfg, log) + + _ = b.RegisterHandler(&handler.ExampleHelloHandler{}) + + messenger, err := b.Build() + if err != nil { + log.Error("failed to build messenger", "error", err) + + return + } + + go func() { + if runErr := messenger.Run(ctx); runErr != nil { + log.Error("messenger run failed", "error", runErr) + + return + } + }() + + messengerBus, err := messenger.GetDefaultBus() + if err != nil { + log.Error("failed to get default bus", "error", err) + + return + } + + _, err = messengerBus.Dispatch(ctx, &message.ExampleHelloMessage{ + Text: "Hello World", + }) + if err != nil { + log.Error("failed to dispatch message", "error", err) + + return + } + + time.Sleep(waitDurationSeconds * time.Second) +} diff --git a/examples/kafka_transport/message/hello.go b/examples/kafka_transport/message/hello.go new file mode 100644 index 0000000..a7877ee --- /dev/null +++ b/examples/kafka_transport/message/hello.go @@ -0,0 +1,9 @@ +package message + +type ExampleHelloMessage struct { + Text string +} + +func (m *ExampleHelloMessage) RoutingKey() string { + return "test_routing_key" +} diff --git a/examples/kafka_transport/messenger.yaml b/examples/kafka_transport/messenger.yaml new file mode 100644 index 0000000..94d5bec --- /dev/null +++ b/examples/kafka_transport/messenger.yaml @@ -0,0 +1,47 @@ +default_bus: default +failure_transport: failed_messages + +buses: + default: ~ + +transports: + kafka: + dsn: "kafka://localhost:29092/" + serializer: default.transport.serializer + options: + topics: + - my-topic + group: my-group + + producer: + async: true + auto_topic_creation: true + required_acks: 1 # -1: all replicas (strongest durability), 0: no ack, 1: leader only + batch_size: 256 + batch_timeout: 10ms + write_timeout: 10s + read_timeout: 10s + balancer: round_robin # least_bytes, hash, round_robin + + consumer: + offset: + type: earliest # earliest, latest, specific + value: 0 # used only for "specific" + commit: + strategy: batch # auto, batch, deferred + interval: 500ms + batch_size: 10 + rebalance: + strategy: roundrobin # roundrobin, range + pool: + size: 3 + min_size: 2 + max_size: 10 + session_timeout: 10s + heartbeat_interval: 2s + + key: + strategy: message_id # none / message_id + +routing: + "*message.ExampleHelloMessage": kafka diff --git a/examples/messenger/messenger.go b/examples/messenger/messenger.go index c0a25c0..98e5a7a 100644 --- a/examples/messenger/messenger.go +++ b/examples/messenger/messenger.go @@ -3,21 +3,25 @@ package main import ( "context" "log/slog" + "os" + "os/signal" + "syscall" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/examples/messenger/handler" "github.com/gerfey/messenger/examples/messenger/message" "github.com/gerfey/messenger/examples/messenger/middleware" ) const ( - waitDurationSeconds = 20 + waitDurationSeconds = 5 ) func main() { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() log := slog.Default() @@ -41,6 +45,9 @@ func main() { return } + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + go func() { if runErr := messenger.Run(ctx); runErr != nil { log.Error("messenger run failed", "error", runErr) @@ -65,5 +72,14 @@ func main() { return } + select { + case sig := <-sigChan: + log.Info("Received signal, shutting down gracefully", "signal", sig) + cancel() + case <-time.After(waitDurationSeconds * time.Second): + log.Info("Timeout reached, shutting down") + cancel() + } + time.Sleep(waitDurationSeconds * time.Second) } diff --git a/examples/messenger/messenger.yaml b/examples/messenger/messenger.yaml index 130078e..79ae45a 100644 --- a/examples/messenger/messenger.yaml +++ b/examples/messenger/messenger.yaml @@ -15,7 +15,10 @@ transports: max_delay: 5s options: auto_setup: true - consumer_pool_size: 10 + pool: + size: 10 + min_size: 5 + max_size: 20 exchange: name: test.exchange type: topic @@ -25,4 +28,4 @@ transports: - test_routing_key routing: - message.ExampleHelloMessage: amqp + "*message.ExampleHelloMessage": amqp diff --git a/examples/redis_transport/handler/hello_handler.go b/examples/redis_transport/handler/hello_handler.go new file mode 100644 index 0000000..0c7cd7d --- /dev/null +++ b/examples/redis_transport/handler/hello_handler.go @@ -0,0 +1,20 @@ +package handler + +import ( + "context" + "fmt" + + "github.com/gerfey/messenger/examples/redis_transport/message" +) + +type ExampleHelloHandler struct{} + +func (u *ExampleHelloHandler) Handle(_ context.Context, msg *message.ExampleHelloMessage) error { + fmt.Printf("Handled: Text=%v\n", msg.Text) + + return nil +} + +func (u *ExampleHelloHandler) GetBusName() string { + return "default" +} diff --git a/examples/redis_transport/message/hello.go b/examples/redis_transport/message/hello.go new file mode 100644 index 0000000..a7877ee --- /dev/null +++ b/examples/redis_transport/message/hello.go @@ -0,0 +1,9 @@ +package message + +type ExampleHelloMessage struct { + Text string +} + +func (m *ExampleHelloMessage) RoutingKey() string { + return "test_routing_key" +} diff --git a/examples/redis_transport/messenger.yaml b/examples/redis_transport/messenger.yaml new file mode 100644 index 0000000..a883c19 --- /dev/null +++ b/examples/redis_transport/messenger.yaml @@ -0,0 +1,32 @@ +default_bus: default +failure_transport: failed_messages + +buses: + default: ~ + +transports: + redis: + dsn: "redis://localhost:6379/0" + retry_strategy: + max_retries: 5 + delay: 500ms + multiplier: 2 + max_delay: 5s + options: + auto_setup: true + stream: stream-messages + group: worker-group + consumer: worker-1 + + failed_messages: + dsn: "amqp://guest:guest@localhost:5672/" + options: + auto_setup: true + exchange: + name: failed_exchange + type: fanout + queues: + failed_messages_queue: ~ + +routing: + "*message.ExampleHelloMessage": redis diff --git a/examples/redis_transport/redis_transport.go b/examples/redis_transport/redis_transport.go new file mode 100644 index 0000000..0791c5f --- /dev/null +++ b/examples/redis_transport/redis_transport.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/examples/redis_transport/handler" + "github.com/gerfey/messenger/examples/redis_transport/message" +) + +const ( + waitDurationSeconds = 20 +) + +func main() { + ctx := context.Background() + + log := slog.Default() + + cfg, err := config.LoadConfig("./examples/redis_transport/messenger.yaml") + if err != nil { + log.Error("ERROR load config", "error", err) + + return + } + + b := builder.NewBuilder(cfg, log) + + _ = b.RegisterHandler(&handler.ExampleHelloHandler{}) + + messenger, err := b.Build() + if err != nil { + log.Error("failed to build messenger", "error", err) + + return + } + + go func() { + if runErr := messenger.Run(ctx); runErr != nil { + log.Error("messenger run failed", "error", runErr) + + return + } + }() + + messengerBus, err := messenger.GetDefaultBus() + if err != nil { + log.Error("failed to get default bus", "error", err) + + return + } + + _, err = messengerBus.Dispatch(ctx, &message.ExampleHelloMessage{ + Text: "Hello World", + }) + if err != nil { + log.Error("failed to dispatch message", "error", err) + + return + } + + time.Sleep(waitDurationSeconds * time.Second) +} diff --git a/examples/retry_messenger/messenger.yaml b/examples/retry_messenger/messenger.yaml index 079887b..9fe82db 100644 --- a/examples/retry_messenger/messenger.yaml +++ b/examples/retry_messenger/messenger.yaml @@ -34,4 +34,4 @@ transports: failed_messages_queue: ~ routing: - message.ExampleHelloMessage: amqp + "*message.ExampleHelloMessage": amqp diff --git a/examples/retry_messenger/retry_messenger.go b/examples/retry_messenger/retry_messenger.go index 162cf2a..9cc3caf 100644 --- a/examples/retry_messenger/retry_messenger.go +++ b/examples/retry_messenger/retry_messenger.go @@ -5,8 +5,8 @@ import ( "log/slog" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/core/event" "github.com/gerfey/messenger/examples/retry_messenger/handler" "github.com/gerfey/messenger/examples/retry_messenger/listener" diff --git a/examples/sync_transport/sync_transport.go b/examples/sync_transport/sync_transport.go index 5bf59f2..b4afdc7 100644 --- a/examples/sync_transport/sync_transport.go +++ b/examples/sync_transport/sync_transport.go @@ -5,8 +5,8 @@ import ( "log/slog" "time" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" "github.com/gerfey/messenger/examples/sync_transport/handler" "github.com/gerfey/messenger/examples/sync_transport/message" ) diff --git a/go.mod b/go.mod index 7456130..e3e3725 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,27 @@ module github.com/gerfey/messenger -go 1.24.3 +go 1.24 require ( github.com/creasty/defaults v1.8.0 + github.com/google/uuid v1.6.0 github.com/ilyakaznacheev/cleanenv v1.5.0 github.com/rabbitmq/amqp091-go v1.10.0 + github.com/redis/go-redis/v9 v9.11.0 + github.com/segmentio/kafka-go v0.4.48 github.com/stretchr/testify v1.10.0 go.uber.org/mock v0.5.2 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/BurntSushi/toml v1.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/joho/godotenv v1.5.1 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index 706d96a..7b3f93e 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,97 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs= +github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= +github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= diff --git a/tests/e2e/happy_path_test.go b/tests/e2e/happy_path_test.go index 5d2f419..9609a39 100644 --- a/tests/e2e/happy_path_test.go +++ b/tests/e2e/happy_path_test.go @@ -10,10 +10,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/tests/fixtures/handlers" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" testHelpers "github.com/gerfey/messenger/tests/helpers" ) diff --git a/tests/e2e/simple_test.go b/tests/e2e/simple_test.go index 8b415c7..c2c4e32 100644 --- a/tests/e2e/simple_test.go +++ b/tests/e2e/simple_test.go @@ -6,10 +6,11 @@ import ( "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/tests/fixtures/handlers" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" testHelpers "github.com/gerfey/messenger/tests/helpers" ) diff --git a/tests/fixtures/configs/config_with_env.yaml b/tests/fixtures/configs/config_with_env.yaml index 169731d..524421f 100644 --- a/tests/fixtures/configs/config_with_env.yaml +++ b/tests/fixtures/configs/config_with_env.yaml @@ -31,4 +31,4 @@ transports: - "#" routing: - "test.TestMessage": default + "*test.TestMessage": default diff --git a/tests/fixtures/configs/e2e.yaml b/tests/fixtures/configs/e2e.yaml index 9e1d75b..b7ce93b 100644 --- a/tests/fixtures/configs/e2e.yaml +++ b/tests/fixtures/configs/e2e.yaml @@ -10,4 +10,4 @@ transports: dsn: "in-memory://" routing: - helpers.TestMessage: in-memory + "*helpers.TestMessage": in-memory diff --git a/tests/fixtures/configs/multiple_transports.yaml b/tests/fixtures/configs/multiple_transports.yaml index 02e2ea7..e966bbc 100644 --- a/tests/fixtures/configs/multiple_transports.yaml +++ b/tests/fixtures/configs/multiple_transports.yaml @@ -12,4 +12,4 @@ transports: dsn: "in-memory://" routing: - helpers.TestMessage: in-memory1 + "*helpers.TestMessage": in-memory1 diff --git a/tests/fixtures/configs/valid_config.yaml b/tests/fixtures/configs/valid_config.yaml index bde7ab1..d46bc7c 100644 --- a/tests/fixtures/configs/valid_config.yaml +++ b/tests/fixtures/configs/valid_config.yaml @@ -25,5 +25,5 @@ transports: consumer_pool_size: 5 routing: - "test.TestMessage": default - "test.AsyncMessage": async + "*test.TestMessage": default + "*test.AsyncMessage": async diff --git a/tests/helpers/messages.go b/tests/helpers/messages.go index 76244ee..e4555bf 100644 --- a/tests/helpers/messages.go +++ b/tests/helpers/messages.go @@ -5,7 +5,6 @@ import ( "errors" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type TestMessage struct { @@ -201,7 +200,7 @@ func (t *TestTransport) Start(_ context.Context) error { return nil } -func (t *TestTransport) Stop() error { +func (t *TestTransport) Close() error { t.IsStopped = true return nil @@ -221,7 +220,7 @@ func (f *TestTransportFactory) Supports(_ string) bool { return true } -func (f *TestTransportFactory) Create(_ string, _ string, _ config.OptionsConfig) (api.Transport, error) { +func (f *TestTransportFactory) Create(_ string, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { if f.CreateError != nil { return nil, f.CreateError } diff --git a/tests/mocks/mock_amqp.go b/tests/mocks/mock_amqp.go deleted file mode 100644 index cac474c..0000000 --- a/tests/mocks/mock_amqp.go +++ /dev/null @@ -1,311 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: interfaces.go -// -// Generated by this command: -// -// mockgen -source=interfaces.go -destination=../../tests/mocks/mock_amqp.go -package=mocks -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - api "github.com/gerfey/messenger/api" - amqp "github.com/gerfey/messenger/transport/amqp" - amqp091 "github.com/rabbitmq/amqp091-go" - gomock "go.uber.org/mock/gomock" -) - -// MockConnectionAMQP is a mock of ConnectionAMQP interface. -type MockConnectionAMQP struct { - ctrl *gomock.Controller - recorder *MockConnectionAMQPMockRecorder - isgomock struct{} -} - -// MockConnectionAMQPMockRecorder is the mock recorder for MockConnectionAMQP. -type MockConnectionAMQPMockRecorder struct { - mock *MockConnectionAMQP -} - -// NewMockConnectionAMQP creates a new mock instance. -func NewMockConnectionAMQP(ctrl *gomock.Controller) *MockConnectionAMQP { - mock := &MockConnectionAMQP{ctrl: ctrl} - mock.recorder = &MockConnectionAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConnectionAMQP) EXPECT() *MockConnectionAMQPMockRecorder { - return m.recorder -} - -// Channel mocks base method. -func (m *MockConnectionAMQP) Channel() (amqp.ChannelAMQP, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Channel") - ret0, _ := ret[0].(amqp.ChannelAMQP) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Channel indicates an expected call of Channel. -func (mr *MockConnectionAMQPMockRecorder) Channel() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Channel", reflect.TypeOf((*MockConnectionAMQP)(nil).Channel)) -} - -// Close mocks base method. -func (m *MockConnectionAMQP) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockConnectionAMQPMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockConnectionAMQP)(nil).Close)) -} - -// IsClosed mocks base method. -func (m *MockConnectionAMQP) IsClosed() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsClosed") - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsClosed indicates an expected call of IsClosed. -func (mr *MockConnectionAMQPMockRecorder) IsClosed() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsClosed", reflect.TypeOf((*MockConnectionAMQP)(nil).IsClosed)) -} - -// MockChannelAMQP is a mock of ChannelAMQP interface. -type MockChannelAMQP struct { - ctrl *gomock.Controller - recorder *MockChannelAMQPMockRecorder - isgomock struct{} -} - -// MockChannelAMQPMockRecorder is the mock recorder for MockChannelAMQP. -type MockChannelAMQPMockRecorder struct { - mock *MockChannelAMQP -} - -// NewMockChannelAMQP creates a new mock instance. -func NewMockChannelAMQP(ctrl *gomock.Controller) *MockChannelAMQP { - mock := &MockChannelAMQP{ctrl: ctrl} - mock.recorder = &MockChannelAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockChannelAMQP) EXPECT() *MockChannelAMQPMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockChannelAMQP) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockChannelAMQPMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockChannelAMQP)(nil).Close)) -} - -// Consume mocks base method. -func (m *MockChannelAMQP) Consume(queue, consumer string, autoAck, exclusive, noLocal, noWait bool, args amqp091.Table) (<-chan amqp091.Delivery, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Consume", queue, consumer, autoAck, exclusive, noLocal, noWait, args) - ret0, _ := ret[0].(<-chan amqp091.Delivery) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Consume indicates an expected call of Consume. -func (mr *MockChannelAMQPMockRecorder) Consume(queue, consumer, autoAck, exclusive, noLocal, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Consume", reflect.TypeOf((*MockChannelAMQP)(nil).Consume), queue, consumer, autoAck, exclusive, noLocal, noWait, args) -} - -// ExchangeDeclare mocks base method. -func (m *MockChannelAMQP) ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp091.Table) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExchangeDeclare", name, kind, durable, autoDelete, internal, noWait, args) - ret0, _ := ret[0].(error) - return ret0 -} - -// ExchangeDeclare indicates an expected call of ExchangeDeclare. -func (mr *MockChannelAMQPMockRecorder) ExchangeDeclare(name, kind, durable, autoDelete, internal, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExchangeDeclare", reflect.TypeOf((*MockChannelAMQP)(nil).ExchangeDeclare), name, kind, durable, autoDelete, internal, noWait, args) -} - -// Publish mocks base method. -func (m *MockChannelAMQP) Publish(exchange, key string, mandatory, immediate bool, msg amqp091.Publishing) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Publish", exchange, key, mandatory, immediate, msg) - ret0, _ := ret[0].(error) - return ret0 -} - -// Publish indicates an expected call of Publish. -func (mr *MockChannelAMQPMockRecorder) Publish(exchange, key, mandatory, immediate, msg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockChannelAMQP)(nil).Publish), exchange, key, mandatory, immediate, msg) -} - -// QueueBind mocks base method. -func (m *MockChannelAMQP) QueueBind(name, key, exchange string, noWait bool, args amqp091.Table) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "QueueBind", name, key, exchange, noWait, args) - ret0, _ := ret[0].(error) - return ret0 -} - -// QueueBind indicates an expected call of QueueBind. -func (mr *MockChannelAMQPMockRecorder) QueueBind(name, key, exchange, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueBind", reflect.TypeOf((*MockChannelAMQP)(nil).QueueBind), name, key, exchange, noWait, args) -} - -// QueueDeclare mocks base method. -func (m *MockChannelAMQP) QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp091.Table) (amqp091.Queue, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "QueueDeclare", name, durable, autoDelete, exclusive, noWait, args) - ret0, _ := ret[0].(amqp091.Queue) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// QueueDeclare indicates an expected call of QueueDeclare. -func (mr *MockChannelAMQPMockRecorder) QueueDeclare(name, durable, autoDelete, exclusive, noWait, args any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueDeclare", reflect.TypeOf((*MockChannelAMQP)(nil).QueueDeclare), name, durable, autoDelete, exclusive, noWait, args) -} - -// MockPublisherAMQP is a mock of PublisherAMQP interface. -type MockPublisherAMQP struct { - ctrl *gomock.Controller - recorder *MockPublisherAMQPMockRecorder - isgomock struct{} -} - -// MockPublisherAMQPMockRecorder is the mock recorder for MockPublisherAMQP. -type MockPublisherAMQPMockRecorder struct { - mock *MockPublisherAMQP -} - -// NewMockPublisherAMQP creates a new mock instance. -func NewMockPublisherAMQP(ctrl *gomock.Controller) *MockPublisherAMQP { - mock := &MockPublisherAMQP{ctrl: ctrl} - mock.recorder = &MockPublisherAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPublisherAMQP) EXPECT() *MockPublisherAMQPMockRecorder { - return m.recorder -} - -// Publish mocks base method. -func (m *MockPublisherAMQP) Publish(ctx context.Context, env api.Envelope) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Publish", ctx, env) - ret0, _ := ret[0].(error) - return ret0 -} - -// Publish indicates an expected call of Publish. -func (mr *MockPublisherAMQPMockRecorder) Publish(ctx, env any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockPublisherAMQP)(nil).Publish), ctx, env) -} - -// MockConsumerAMQP is a mock of ConsumerAMQP interface. -type MockConsumerAMQP struct { - ctrl *gomock.Controller - recorder *MockConsumerAMQPMockRecorder - isgomock struct{} -} - -// MockConsumerAMQPMockRecorder is the mock recorder for MockConsumerAMQP. -type MockConsumerAMQPMockRecorder struct { - mock *MockConsumerAMQP -} - -// NewMockConsumerAMQP creates a new mock instance. -func NewMockConsumerAMQP(ctrl *gomock.Controller) *MockConsumerAMQP { - mock := &MockConsumerAMQP{ctrl: ctrl} - mock.recorder = &MockConsumerAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConsumerAMQP) EXPECT() *MockConsumerAMQPMockRecorder { - return m.recorder -} - -// Consume mocks base method. -func (m *MockConsumerAMQP) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Consume", ctx, handler) - ret0, _ := ret[0].(error) - return ret0 -} - -// Consume indicates an expected call of Consume. -func (mr *MockConsumerAMQPMockRecorder) Consume(ctx, handler any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Consume", reflect.TypeOf((*MockConsumerAMQP)(nil).Consume), ctx, handler) -} - -// MockRetryAMQP is a mock of RetryAMQP interface. -type MockRetryAMQP struct { - ctrl *gomock.Controller - recorder *MockRetryAMQPMockRecorder - isgomock struct{} -} - -// MockRetryAMQPMockRecorder is the mock recorder for MockRetryAMQP. -type MockRetryAMQPMockRecorder struct { - mock *MockRetryAMQP -} - -// NewMockRetryAMQP creates a new mock instance. -func NewMockRetryAMQP(ctrl *gomock.Controller) *MockRetryAMQP { - mock := &MockRetryAMQP{ctrl: ctrl} - mock.recorder = &MockRetryAMQPMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRetryAMQP) EXPECT() *MockRetryAMQPMockRecorder { - return m.recorder -} - -// Retry mocks base method. -func (m *MockRetryAMQP) Retry(ctx context.Context, env api.Envelope) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Retry", ctx, env) - ret0, _ := ret[0].(error) - return ret0 -} - -// Retry indicates an expected call of Retry. -func (mr *MockRetryAMQPMockRecorder) Retry(ctx, env any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Retry", reflect.TypeOf((*MockRetryAMQP)(nil).Retry), ctx, env) -} diff --git a/tests/mocks/mock_transport.go b/tests/mocks/mock_transport.go index 83fdd92..6ed59ae 100644 --- a/tests/mocks/mock_transport.go +++ b/tests/mocks/mock_transport.go @@ -14,7 +14,6 @@ import ( reflect "reflect" api "github.com/gerfey/messenger/api" - config "github.com/gerfey/messenger/config" gomock "go.uber.org/mock/gomock" ) @@ -42,6 +41,20 @@ func (m *MockTransport) EXPECT() *MockTransportMockRecorder { return m.recorder } +// Close mocks base method. +func (m *MockTransport) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockTransportMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTransport)(nil).Close)) +} + // Name mocks base method. func (m *MockTransport) Name() string { m.ctrl.T.Helper() @@ -174,6 +187,148 @@ func (mr *MockReceiverMockRecorder) Receive(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Receive", reflect.TypeOf((*MockReceiver)(nil).Receive), arg0, arg1) } +// MockCloser is a mock of Closer interface. +type MockCloser struct { + ctrl *gomock.Controller + recorder *MockCloserMockRecorder + isgomock struct{} +} + +// MockCloserMockRecorder is the mock recorder for MockCloser. +type MockCloserMockRecorder struct { + mock *MockCloser +} + +// NewMockCloser creates a new mock instance. +func NewMockCloser(ctrl *gomock.Controller) *MockCloser { + mock := &MockCloser{ctrl: ctrl} + mock.recorder = &MockCloserMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCloser) EXPECT() *MockCloserMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockCloser) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockCloserMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockCloser)(nil).Close)) +} + +// MockProducer is a mock of Producer interface. +type MockProducer struct { + ctrl *gomock.Controller + recorder *MockProducerMockRecorder + isgomock struct{} +} + +// MockProducerMockRecorder is the mock recorder for MockProducer. +type MockProducerMockRecorder struct { + mock *MockProducer +} + +// NewMockProducer creates a new mock instance. +func NewMockProducer(ctrl *gomock.Controller) *MockProducer { + mock := &MockProducer{ctrl: ctrl} + mock.recorder = &MockProducerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockProducer) EXPECT() *MockProducerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockProducer) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockProducerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockProducer)(nil).Close)) +} + +// Send mocks base method. +func (m *MockProducer) Send(arg0 context.Context, arg1 api.Envelope) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockProducerMockRecorder) Send(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockProducer)(nil).Send), arg0, arg1) +} + +// MockConsumer is a mock of Consumer interface. +type MockConsumer struct { + ctrl *gomock.Controller + recorder *MockConsumerMockRecorder + isgomock struct{} +} + +// MockConsumerMockRecorder is the mock recorder for MockConsumer. +type MockConsumerMockRecorder struct { + mock *MockConsumer +} + +// NewMockConsumer creates a new mock instance. +func NewMockConsumer(ctrl *gomock.Controller) *MockConsumer { + mock := &MockConsumer{ctrl: ctrl} + mock.recorder = &MockConsumerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConsumer) EXPECT() *MockConsumerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockConsumer) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockConsumerMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockConsumer)(nil).Close)) +} + +// Consume mocks base method. +func (m *MockConsumer) Consume(arg0 context.Context, arg1 func(context.Context, api.Envelope) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Consume", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Consume indicates an expected call of Consume. +func (mr *MockConsumerMockRecorder) Consume(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Consume", reflect.TypeOf((*MockConsumer)(nil).Consume), arg0, arg1) +} + // MockRetryableTransport is a mock of RetryableTransport interface. type MockRetryableTransport struct { ctrl *gomock.Controller @@ -198,6 +353,20 @@ func (m *MockRetryableTransport) EXPECT() *MockRetryableTransportMockRecorder { return m.recorder } +// Close mocks base method. +func (m *MockRetryableTransport) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockRetryableTransportMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockRetryableTransport)(nil).Close)) +} + // Name mocks base method. func (m *MockRetryableTransport) Name() string { m.ctrl.T.Helper() @@ -254,6 +423,100 @@ func (mr *MockRetryableTransportMockRecorder) Send(arg0, arg1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockRetryableTransport)(nil).Send), arg0, arg1) } +// MockSetupableTransport is a mock of SetupableTransport interface. +type MockSetupableTransport struct { + ctrl *gomock.Controller + recorder *MockSetupableTransportMockRecorder + isgomock struct{} +} + +// MockSetupableTransportMockRecorder is the mock recorder for MockSetupableTransport. +type MockSetupableTransportMockRecorder struct { + mock *MockSetupableTransport +} + +// NewMockSetupableTransport creates a new mock instance. +func NewMockSetupableTransport(ctrl *gomock.Controller) *MockSetupableTransport { + mock := &MockSetupableTransport{ctrl: ctrl} + mock.recorder = &MockSetupableTransportMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSetupableTransport) EXPECT() *MockSetupableTransportMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockSetupableTransport) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockSetupableTransportMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSetupableTransport)(nil).Close)) +} + +// Name mocks base method. +func (m *MockSetupableTransport) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockSetupableTransportMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockSetupableTransport)(nil).Name)) +} + +// Receive mocks base method. +func (m *MockSetupableTransport) Receive(arg0 context.Context, arg1 func(context.Context, api.Envelope) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Receive", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Receive indicates an expected call of Receive. +func (mr *MockSetupableTransportMockRecorder) Receive(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Receive", reflect.TypeOf((*MockSetupableTransport)(nil).Receive), arg0, arg1) +} + +// Send mocks base method. +func (m *MockSetupableTransport) Send(arg0 context.Context, arg1 api.Envelope) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockSetupableTransportMockRecorder) Send(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSetupableTransport)(nil).Send), arg0, arg1) +} + +// Setup mocks base method. +func (m *MockSetupableTransport) Setup(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Setup", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Setup indicates an expected call of Setup. +func (mr *MockSetupableTransportMockRecorder) Setup(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Setup", reflect.TypeOf((*MockSetupableTransport)(nil).Setup), ctx) +} + // MockSenderLocator is a mock of SenderLocator interface. type MockSenderLocator struct { ctrl *gomock.Controller @@ -355,18 +618,18 @@ func (m *MockTransportFactory) EXPECT() *MockTransportFactoryMockRecorder { } // Create mocks base method. -func (m *MockTransportFactory) Create(arg0, arg1 string, arg2 config.OptionsConfig) (api.Transport, error) { +func (m *MockTransportFactory) Create(arg0, arg1 string, arg2 []byte, arg3 api.Serializer) (api.Transport, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(api.Transport) ret1, _ := ret[1].(error) return ret0, ret1 } // Create indicates an expected call of Create. -func (mr *MockTransportFactoryMockRecorder) Create(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockTransportFactoryMockRecorder) Create(arg0, arg1, arg2, arg3 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockTransportFactory)(nil).Create), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockTransportFactory)(nil).Create), arg0, arg1, arg2, arg3) } // Supports mocks base method. diff --git a/transport/amqp/adapters.go b/transport/amqp/adapters.go deleted file mode 100644 index 6f5c25d..0000000 --- a/transport/amqp/adapters.go +++ /dev/null @@ -1,118 +0,0 @@ -package amqp - -import ( - "context" - - amqp "github.com/rabbitmq/amqp091-go" - - "github.com/gerfey/messenger/api" -) - -type ConnectionAdapter struct { - conn *amqp.Connection -} - -func NewConnectionAdapter(conn *amqp.Connection) ConnectionAMQP { - return &ConnectionAdapter{conn: conn} -} - -func (c *ConnectionAdapter) Channel() (ChannelAMQP, error) { - ch, err := c.conn.Channel() - if err != nil { - return nil, err - } - - return NewChannelAdapter(ch), nil -} - -func (c *ConnectionAdapter) IsClosed() bool { - return c.conn.IsClosed() -} - -func (c *ConnectionAdapter) Close() error { - return c.conn.Close() -} - -type ChannelAdapter struct { - ch *amqp.Channel -} - -func NewChannelAdapter(ch *amqp.Channel) ChannelAMQP { - return &ChannelAdapter{ch: ch} -} - -func (c *ChannelAdapter) ExchangeDeclare( - name, kind string, - durable, autoDelete, internal, noWait bool, - args amqp.Table, -) error { - return c.ch.ExchangeDeclare( - name, kind, - durable, autoDelete, internal, noWait, - args, - ) -} - -func (c *ChannelAdapter) QueueDeclare( - name string, - durable, autoDelete, exclusive, noWait bool, - args amqp.Table, -) (amqp.Queue, error) { - return c.ch.QueueDeclare(name, durable, autoDelete, exclusive, noWait, args) -} - -func (c *ChannelAdapter) QueueBind(name, key, exchange string, noWait bool, args amqp.Table) error { - return c.ch.QueueBind(name, key, exchange, noWait, args) -} - -func (c *ChannelAdapter) Publish(exchange, key string, mandatory, immediate bool, msg amqp.Publishing) error { - return c.ch.Publish(exchange, key, mandatory, immediate, msg) -} - -func (c *ChannelAdapter) Consume( - queue, consumer string, - autoAck, exclusive, noLocal, noWait bool, - args amqp.Table, -) (<-chan amqp.Delivery, error) { - return c.ch.Consume(queue, consumer, autoAck, exclusive, noLocal, noWait, args) -} - -func (c *ChannelAdapter) Close() error { - return c.ch.Close() -} - -type PublisherAdapter struct { - publisher *Publisher -} - -func NewPublisherAdapter(publisher *Publisher) PublisherAMQP { - return &PublisherAdapter{publisher: publisher} -} - -func (p *PublisherAdapter) Publish(ctx context.Context, env api.Envelope) error { - return p.publisher.Publish(ctx, env) -} - -type ConsumerAdapter struct { - consumer *Consumer -} - -func NewConsumerAdapter(consumer *Consumer) ConsumerAMQP { - return &ConsumerAdapter{consumer: consumer} -} - -func (c *ConsumerAdapter) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - return c.consumer.Consume(ctx, handler) -} - -type RetryAdapter struct { - retry *Retry -} - -func NewRetryAdapter(retry *Retry) RetryAMQP { - return &RetryAdapter{retry: retry} -} - -func (r *RetryAdapter) Retry(ctx context.Context, env api.Envelope) error { - return r.retry.Retry(ctx, env) -} diff --git a/transport/amqp/benchmark_test.go b/transport/amqp/benchmark_test.go new file mode 100644 index 0000000..5937161 --- /dev/null +++ b/transport/amqp/benchmark_test.go @@ -0,0 +1,130 @@ +package amqp_test + +import ( + "fmt" + "log/slog" + "sync" + "testing" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" +) + +const ( + benchmarkDSN = "amqp://guest:guest@localhost:5672/" +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func (m *BenchmarkMessage) RoutingKey() string { + return "benchmark_routing_key" +} + +func setupMessenger(b *testing.B) api.MessageBus { + b.Helper() + + logger := slog.New(slog.DiscardHandler) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "amqp": { + DSN: benchmarkDSN, + Serializer: "default.transport.serializer", + Options: map[string]any{ + "auto_setup": true, + "exchange": map[string]any{ + "name": "benchmark_exchange", + "type": "topic", + }, + "queues": map[string]any{ + "benchmark_queue": map[string]any{ + "binding_keys": []string{"benchmark_routing_key"}, + }, + }, + }, + }, + }, + Routing: map[string]string{ + "*amqp_test.BenchmarkMessage": "amqp", + }, + } + + builderInstance := builder.NewBuilder(cfg, logger) + + builderInstance.RegisterMessage(&BenchmarkMessage{}) + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, size, false) + }) + } +} diff --git a/transport/amqp/config.go b/transport/amqp/config.go index 955c163..fcf5c62 100644 --- a/transport/amqp/config.go +++ b/transport/amqp/config.go @@ -1,11 +1,35 @@ package amqp -import ( - "github.com/gerfey/messenger/config" -) - type TransportConfig struct { Name string DSN string - Options config.OptionsConfig + Options OptionsConfig +} + +type OptionsConfig struct { + AutoSetup bool `yaml:"auto_setup" default:"false"` + Pool PoolConfig `yaml:"pool"` + Exchange ExchangeConfig `yaml:"exchange"` + Queues map[string]Queue `yaml:"queues"` +} + +type PoolConfig struct { + Size int `yaml:"size" default:"10"` + MinSize int `yaml:"min_size" default:"5"` + MaxSize int `yaml:"max_size" default:"20"` +} + +type ExchangeConfig struct { + Name string `yaml:"name"` + Type string `yaml:"type" default:"topic"` // topic, direct, fanout + Durable bool `yaml:"durable" default:"true"` + AutoDelete bool `yaml:"auto_delete" default:"false"` + Internal bool `yaml:"internal" default:"false"` +} + +type Queue struct { + BindingKeys []string `yaml:"binding_keys"` + Durable bool `yaml:"durable" default:"true"` + Exclusive bool `yaml:"exclusive" default:"false"` + AutoDelete bool `yaml:"auto_delete" default:"false"` } diff --git a/transport/amqp/connection.go b/transport/amqp/connection.go index 9034fae..c6fb1d5 100644 --- a/transport/amqp/connection.go +++ b/transport/amqp/connection.go @@ -13,9 +13,9 @@ type Connection struct { lock sync.Mutex } -func NewConnection(dsn string) (*Connection, error) { +func NewConnection(dsn string) (ConnectionAMQP, error) { conn := &Connection{dsn: dsn} - err := conn.connect() + err := conn.Connect() if err != nil { return nil, err } @@ -28,7 +28,7 @@ func (c *Connection) Channel() (*amqp.Channel, error) { defer c.lock.Unlock() if c.conn == nil || c.conn.IsClosed() { - if err := c.connect(); err != nil { + if err := c.Connect(); err != nil { return nil, err } } @@ -41,7 +41,7 @@ func (c *Connection) Channel() (*amqp.Channel, error) { return ch, nil } -func (c *Connection) connect() error { +func (c *Connection) Connect() error { c.lock.Lock() defer c.lock.Unlock() @@ -54,3 +54,11 @@ func (c *Connection) connect() error { return nil } + +func (c *Connection) Close() error { + return c.conn.Close() +} + +func (c *Connection) IsConnect() bool { + return c.conn != nil && !c.conn.IsClosed() +} diff --git a/transport/amqp/consumer.go b/transport/amqp/consumer.go index a4aae18..b7006c7 100644 --- a/transport/amqp/consumer.go +++ b/transport/amqp/consumer.go @@ -2,7 +2,9 @@ package amqp import ( "context" + "errors" "fmt" + "sync" amqp "github.com/rabbitmq/amqp091-go" @@ -10,22 +12,31 @@ import ( "github.com/gerfey/messenger/core/stamps" ) +const ( + defaultPoolSize = 10 +) + type Consumer struct { - conn *Connection - cfg TransportConfig + config TransportConfig + connection ConnectionAMQP serializer api.Serializer + wg sync.WaitGroup } -func NewConsumer(conn *Connection, cfg TransportConfig, serializer api.Serializer) *Consumer { +func NewConsumer(config TransportConfig, connection ConnectionAMQP, serializer api.Serializer) (api.Consumer, error) { return &Consumer{ - conn: conn, - cfg: cfg, + config: config, + connection: connection, serializer: serializer, - } + }, nil } func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { - ch, err := c.conn.Channel() + if !c.connection.IsConnect() { + return errors.New("amqp connection is not available") + } + + ch, err := c.connection.Channel() if err != nil { return fmt.Errorf("failed to create AMQP channel for consumer: %w", err) } @@ -42,31 +53,54 @@ func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, ap } <-ctx.Done() + close(jobs) + c.wg.Wait() return ctx.Err() } +func (c *Consumer) Close() error { + return nil +} + func (c *Consumer) startWorkerPool( ctx context.Context, jobs chan job, handler func(context.Context, api.Envelope) error, ) { - poolSize := c.cfg.Options.ConsumerPoolSize + poolSize := c.config.Options.Pool.Size if poolSize <= 0 { - poolSize = 10 + poolSize = defaultPoolSize } for range poolSize { - go func() { - for j := range jobs { - c.handleDelivery(ctx, j.d, handler) + c.wg.Add(1) + go c.startWorker(ctx, jobs, handler) + } +} + +func (c *Consumer) startWorker( + ctx context.Context, + jobs chan job, + handler func(context.Context, api.Envelope) error, +) { + defer c.wg.Done() + + for { + select { + case <-ctx.Done(): + return + case j, ok := <-jobs: + if !ok { + return } - }() + c.handleDelivery(ctx, j.d, handler) + } } } func (c *Consumer) startQueueConsumers(ctx context.Context, ch *amqp.Channel, jobs chan job) error { - for queueName := range c.cfg.Options.Queues { + for queueName := range c.config.Options.Queues { msgs, consumeErr := ch.ConsumeWithContext( ctx, queueName, @@ -91,8 +125,6 @@ func (c *Consumer) processQueueMessages(ctx context.Context, jobs chan job, mess for { select { case <-ctx.Done(): - close(jobs) - return case d, ok := <-messages: if !ok { @@ -123,7 +155,7 @@ func (c *Consumer) handleDelivery( } env = env.WithStamp(stamps.ReceivedStamp{ - Transport: c.cfg.Name, + Transport: c.config.Name, }) err = handler(ctx, env) diff --git a/transport/amqp/factory.go b/transport/amqp/factory.go index c0c2513..1cb0275 100644 --- a/transport/amqp/factory.go +++ b/transport/amqp/factory.go @@ -1,35 +1,45 @@ package amqp import ( - "log/slog" + "fmt" "strings" + "github.com/creasty/defaults" + "gopkg.in/yaml.v3" + "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) -type TransportFactory struct { - logger *slog.Logger - resolver api.TypeResolver -} +type TransportFactory struct{} -func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { - return &TransportFactory{ - logger: logger, - resolver: resolver, - } +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} } func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "amqp://") } -func (f *TransportFactory) Create(name string, dsn string, options config.OptionsConfig) (api.Transport, error) { +func (f *TransportFactory) Create( + name string, + dsn string, + options []byte, + serializer api.Serializer, +) (api.Transport, error) { + var opts OptionsConfig + if err := defaults.Set(&opts); err != nil { + return nil, fmt.Errorf("set defaults: %w", err) + } + + if err := yaml.Unmarshal(options, &opts); err != nil { + return nil, fmt.Errorf("unmarshal options: %w", err) + } + cfg := TransportConfig{ Name: name, DSN: dsn, - Options: options, + Options: opts, } - return NewTransport(cfg, f.resolver, f.logger) + return NewTransport(cfg, serializer) } diff --git a/transport/amqp/factory_test.go b/transport/amqp/factory_test.go index 59f128a..d21aad6 100644 --- a/transport/amqp/factory_test.go +++ b/transport/amqp/factory_test.go @@ -1,14 +1,15 @@ package amqp_test import ( - "log/slog" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/core/serializer" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/amqp" ) @@ -17,10 +18,7 @@ func TestNewTransportFactory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - - factory := amqp.NewTransportFactory(logger, mockResolver) + factory := amqp.NewTransportFactory() assert.NotNil(t, factory) assert.IsType(t, &amqp.TransportFactory{}, factory) @@ -64,9 +62,7 @@ func TestTransportFactory_Supports(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := amqp.NewTransportFactory(logger, mockResolver) + factory := amqp.NewTransportFactory() got := factory.Supports(tt.dsn) assert.Equal(t, tt.want, got) @@ -78,23 +74,22 @@ func TestTransportFactory_Create(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := amqp.NewTransportFactory(logger, mockResolver) + factory := amqp.NewTransportFactory() name := "test-amqp" dsn := "amqp://guest:guest@non-existent-host:5672/" - options := config.OptionsConfig{ + options := amqp.OptionsConfig{ AutoSetup: true, - Exchange: config.ExchangeConfig{ + Exchange: amqp.ExchangeConfig{ Name: "test-exchange", Type: "direct", Durable: true, AutoDelete: false, Internal: false, }, - Queues: map[string]config.Queue{ + Queues: map[string]amqp.Queue{ "test-queue": { BindingKeys: []string{"test-key"}, Durable: true, @@ -104,7 +99,12 @@ func TestTransportFactory_Create(t *testing.T) { }, } - _, err := factory.Create(name, dsn, options) + ser := serializer.NewSerializer(mockResolver) + + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) + + _, err = factory.Create(name, dsn, optionsBytes, ser) require.Error(t, err) assert.Contains(t, err.Error(), "failed to connect") diff --git a/transport/amqp/interfaces.go b/transport/amqp/interfaces.go deleted file mode 100644 index 29bc927..0000000 --- a/transport/amqp/interfaces.go +++ /dev/null @@ -1,42 +0,0 @@ -package amqp - -import ( - "context" - - amqp "github.com/rabbitmq/amqp091-go" - - "github.com/gerfey/messenger/api" -) - -//go:generate go run go.uber.org/mock/mockgen@latest -source=interfaces.go -destination=../../tests/mocks/mock_amqp.go -package=mocks - -type ConnectionAMQP interface { - Channel() (ChannelAMQP, error) - IsClosed() bool - Close() error -} - -type ChannelAMQP interface { - ExchangeDeclare(name, kind string, durable, autoDelete, internal, noWait bool, args amqp.Table) error - QueueDeclare(name string, durable, autoDelete, exclusive, noWait bool, args amqp.Table) (amqp.Queue, error) - QueueBind(name, key, exchange string, noWait bool, args amqp.Table) error - Publish(exchange, key string, mandatory, immediate bool, msg amqp.Publishing) error - Consume( - queue, consumer string, - autoAck, exclusive, noLocal, noWait bool, - args amqp.Table, - ) (<-chan amqp.Delivery, error) - Close() error -} - -type PublisherAMQP interface { - Publish(ctx context.Context, env api.Envelope) error -} - -type ConsumerAMQP interface { - Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error -} - -type RetryAMQP interface { - Retry(ctx context.Context, env api.Envelope) error -} diff --git a/transport/amqp/base_publisher.go b/transport/amqp/producer.go similarity index 56% rename from transport/amqp/base_publisher.go rename to transport/amqp/producer.go index 6959ea8..58eca9f 100644 --- a/transport/amqp/base_publisher.go +++ b/transport/amqp/producer.go @@ -2,6 +2,7 @@ package amqp import ( "context" + "errors" "fmt" amqp "github.com/rabbitmq/amqp091-go" @@ -9,22 +10,26 @@ import ( "github.com/gerfey/messenger/api" ) -type BasePublisher struct { - conn *Connection - cfg TransportConfig +type Producer struct { + config TransportConfig + connection ConnectionAMQP serializer api.Serializer } -func NewBasePublisher(conn *Connection, cfg TransportConfig, serializer api.Serializer) *BasePublisher { - return &BasePublisher{ - conn: conn, - cfg: cfg, +func NewProducer(config TransportConfig, connection ConnectionAMQP, serializer api.Serializer) (api.Producer, error) { + return &Producer{ + config: config, + connection: connection, serializer: serializer, - } + }, nil } -func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) error { - body, headersMap, err := bp.serializer.Marshal(env) +func (p *Producer) Send(ctx context.Context, env api.Envelope) error { + if !p.connection.IsConnect() { + return errors.New("amqp connection is not available") + } + + body, headersMap, err := p.serializer.Marshal(env) if err != nil { return fmt.Errorf("failed to marshal envelope: %w", err) } @@ -34,7 +39,7 @@ func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) e headers[k] = v } - ch, err := bp.conn.Channel() + ch, err := p.connection.Channel() if err != nil { return fmt.Errorf("failed to create AMQP channel: %w", err) } @@ -45,7 +50,7 @@ func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) e routingKey := getRoutingKey(env.Message()) err = ch.PublishWithContext(ctx, - bp.cfg.Options.Exchange.Name, + p.config.Options.Exchange.Name, routingKey, false, false, @@ -57,7 +62,7 @@ func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) e if err != nil { return fmt.Errorf( "failed to publish message to exchange '%s' with routing key '%s': %w", - bp.cfg.Options.Exchange.Name, + p.config.Options.Exchange.Name, routingKey, err, ) @@ -65,3 +70,7 @@ func (bp *BasePublisher) PublishMessage(ctx context.Context, env api.Envelope) e return nil } + +func (p *Producer) Close() error { + return nil +} diff --git a/transport/amqp/publisher.go b/transport/amqp/publisher.go deleted file mode 100644 index 3186835..0000000 --- a/transport/amqp/publisher.go +++ /dev/null @@ -1,21 +0,0 @@ -package amqp - -import ( - "context" - - "github.com/gerfey/messenger/api" -) - -type Publisher struct { - *BasePublisher -} - -func NewPublisher(conn *Connection, cfg TransportConfig, serializer api.Serializer) *Publisher { - return &Publisher{ - BasePublisher: NewBasePublisher(conn, cfg, serializer), - } -} - -func (p *Publisher) Publish(ctx context.Context, env api.Envelope) error { - return p.PublishMessage(ctx, env) -} diff --git a/transport/amqp/retry.go b/transport/amqp/retry.go deleted file mode 100644 index a2f79b3..0000000 --- a/transport/amqp/retry.go +++ /dev/null @@ -1,21 +0,0 @@ -package amqp - -import ( - "context" - - "github.com/gerfey/messenger/api" -) - -type Retry struct { - *BasePublisher -} - -func NewRetry(conn *Connection, cfg TransportConfig, serializer api.Serializer) *Retry { - return &Retry{ - BasePublisher: NewBasePublisher(conn, cfg, serializer), - } -} - -func (r *Retry) Retry(ctx context.Context, env api.Envelope) error { - return r.PublishMessage(ctx, env) -} diff --git a/transport/amqp/transport.go b/transport/amqp/transport.go index f85a34d..cf5ad75 100644 --- a/transport/amqp/transport.go +++ b/transport/amqp/transport.go @@ -3,66 +3,60 @@ package amqp import ( "context" "fmt" - "log/slog" "reflect" + amqp "github.com/rabbitmq/amqp091-go" + "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/serializer" ) -type Transport struct { - cfg TransportConfig - publisher *Publisher - consumer *Consumer - retry *Retry - serializer api.Serializer - conn *Connection - logger *slog.Logger +type ConnectionAMQP interface { + Channel() (*amqp.Channel, error) + Connect() error + IsConnect() bool + Close() error } -func NewTransport(cfg TransportConfig, resolver api.TypeResolver, logger *slog.Logger) (api.Transport, error) { - conn, err := NewConnection(cfg.DSN) - if err != nil { - logger.Error("failed to create AMQP connection", "dsn", cfg.DSN, "error", err) +type Transport struct { + config TransportConfig + producer api.Producer + consumer api.Consumer + connection ConnectionAMQP +} - return nil, err +func NewTransport( + config TransportConfig, + serializer api.Serializer, +) (api.Transport, error) { + connection, errConnection := NewConnection(config.DSN) + if errConnection != nil { + return nil, fmt.Errorf("failed to create connection: %w", errConnection) } - ser := serializer.NewSerializer(resolver) - - pub := NewPublisher(conn, cfg, ser) - cons := NewConsumer(conn, cfg, ser) - ret := NewRetry(conn, cfg, ser) - - transport := &Transport{ - cfg: cfg, - publisher: pub, - consumer: cons, - retry: ret, - serializer: ser, - conn: conn, - logger: logger, + producer, errProducer := NewProducer(config, connection, serializer) + if errProducer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errProducer) } - if cfg.Options.AutoSetup { - if setupErr := transport.setup(); setupErr != nil { - logger.Error("failed to setup AMQP transport", "transport", cfg.Name, "error", setupErr) - - return nil, setupErr - } - - logger.Debug("AMQP transport setup completed", "transport", cfg.Name) + consumer, errConsumer := NewConsumer(config, connection, serializer) + if errConsumer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errConsumer) } - return transport, nil + return &Transport{ + config: config, + producer: producer, + consumer: consumer, + connection: connection, + }, nil } func (t *Transport) Name() string { - return t.cfg.Name + return t.config.Name } func (t *Transport) Send(ctx context.Context, env api.Envelope) error { - return t.publisher.Publish(ctx, env) + return t.producer.Send(ctx, env) } func (t *Transport) Receive(ctx context.Context, handler func(context.Context, api.Envelope) error) error { @@ -70,14 +64,16 @@ func (t *Transport) Receive(ctx context.Context, handler func(context.Context, a } func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { - return t.retry.Retry(ctx, env) + return t.producer.Send(ctx, env) } -func (t *Transport) setup() error { - ch, err := t.conn.Channel() - if err != nil { - t.logger.Error("failed to open channel", "error", err) +func (t *Transport) Setup(_ context.Context) error { + if !t.config.Options.AutoSetup { + return nil + } + ch, err := t.connection.Channel() + if err != nil { return fmt.Errorf("failed to open channel: %w", err) } defer func() { @@ -85,21 +81,19 @@ func (t *Transport) setup() error { }() err = ch.ExchangeDeclare( - t.cfg.Options.Exchange.Name, - t.cfg.Options.Exchange.Type, - t.cfg.Options.Exchange.Durable, - t.cfg.Options.Exchange.AutoDelete, - t.cfg.Options.Exchange.Internal, + t.config.Options.Exchange.Name, + t.config.Options.Exchange.Type, + t.config.Options.Exchange.Durable, + t.config.Options.Exchange.AutoDelete, + t.config.Options.Exchange.Internal, false, nil, ) if err != nil { - t.logger.Error("failed to declare exchange", "exchange", t.cfg.Options.Exchange.Name, "error", err) - return fmt.Errorf("failed to declare exchange: %w", err) } - for queueName, queueCfg := range t.cfg.Options.Queues { + for queueName, queueCfg := range t.config.Options.Queues { _, err = ch.QueueDeclare( queueName, queueCfg.Durable, @@ -109,8 +103,6 @@ func (t *Transport) setup() error { nil, ) if err != nil { - t.logger.Error("declare queue", "queue", queueName, "error", err) - return fmt.Errorf("declare queue: %w", err) } @@ -124,13 +116,11 @@ func (t *Transport) setup() error { bindErr := ch.QueueBind( queueName, bindingKey, - t.cfg.Options.Exchange.Name, + t.config.Options.Exchange.Name, false, nil, ) if bindErr != nil { - t.logger.Error("bind queue", "queue", queueName, "binding_key", bindingKey, "error", bindErr) - return fmt.Errorf("bind queue: %w", bindErr) } } @@ -139,6 +129,14 @@ func (t *Transport) setup() error { return nil } +func (t *Transport) Close() error { + if err := t.connection.Close(); err != nil { + return fmt.Errorf("failed to close connection: %w", err) + } + + return nil +} + func getRoutingKey(msg any) string { var routingKey string if rk, ok := msg.(api.RoutedMessage); ok { diff --git a/transport/amqp/transport_test.go b/transport/amqp/transport_test.go deleted file mode 100644 index 1201a64..0000000 --- a/transport/amqp/transport_test.go +++ /dev/null @@ -1,271 +0,0 @@ -package amqp_test - -import ( - "context" - "errors" - "testing" - "time" - - amqp091 "github.com/rabbitmq/amqp091-go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - - "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/builder" - "github.com/gerfey/messenger/config" - "github.com/gerfey/messenger/core/envelope" - "github.com/gerfey/messenger/tests/helpers" - "github.com/gerfey/messenger/tests/mocks" - "github.com/gerfey/messenger/transport/amqp" -) - -func TestTransport_Name(t *testing.T) { - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: false, - }, - } - - resolver := builder.NewResolver() - logger, _ := helpers.NewFakeLogger() - - transport, err := amqp.NewTransport(cfg, resolver, logger) - if err != nil { - t.Skip("Skipping test - requires AMQP connection") - - return - } - - assert.Equal(t, "test-transport", transport.Name()) -} - -func TestTransport_Setup_WithMocks(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - t.Run("successful setup", func(t *testing.T) { - mockConn := mocks.NewMockConnectionAMQP(ctrl) - mockChannel := mocks.NewMockChannelAMQP(ctrl) - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: true, - Exchange: config.ExchangeConfig{ - Name: "test-exchange", - Type: "direct", - Durable: true, - AutoDelete: false, - Internal: false, - }, - Queues: map[string]config.Queue{ - "test-queue": { - Durable: true, - AutoDelete: false, - Exclusive: false, - BindingKeys: []string{"test.key"}, - }, - }, - }, - } - - mockConn.EXPECT().Channel().Return(mockChannel, nil) - mockChannel.EXPECT().ExchangeDeclare( - "test-exchange", "direct", true, false, false, false, nil, - ).Return(nil) - mockChannel.EXPECT().QueueDeclare( - "test-queue", true, false, false, false, nil, - ).Return(amqp091.Queue{Name: "test-queue"}, nil) - mockChannel.EXPECT().QueueBind( - "test-queue", "test.key", "test-exchange", false, nil, - ).Return(nil) - mockChannel.EXPECT().Close().Return(nil) - - transport := createMockTransport(cfg, mockConn) - err := transport.Setup() - require.NoError(t, err) - }) - - t.Run("setup with channel error", func(t *testing.T) { - mockConn := mocks.NewMockConnectionAMQP(ctrl) - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: true, - }, - } - - expectedErr := errors.New("channel error") - mockConn.EXPECT().Channel().Return(nil, expectedErr) - - transport := createMockTransport(cfg, mockConn) - err := transport.Setup() - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to open channel") - }) - - t.Run("setup with exchange declare error", func(t *testing.T) { - mockConn := mocks.NewMockConnectionAMQP(ctrl) - mockChannel := mocks.NewMockChannelAMQP(ctrl) - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: true, - Exchange: config.ExchangeConfig{ - Name: "test-exchange", - Type: "direct", - }, - }, - } - - expectedErr := errors.New("exchange error") - mockConn.EXPECT().Channel().Return(mockChannel, nil) - mockChannel.EXPECT().ExchangeDeclare( - gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), - gomock.Any(), gomock.Any(), gomock.Any(), - ).Return(expectedErr) - mockChannel.EXPECT().Close().Return(nil) - - transport := createMockTransport(cfg, mockConn) - err := transport.Setup() - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to declare exchange") - }) -} - -func TestTransport_Integration_Methods(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - cfg := amqp.TransportConfig{ - Name: "test-transport", - DSN: "amqp://localhost", - Options: config.OptionsConfig{ - AutoSetup: false, - }, - } - - resolver := builder.NewResolver() - resolver.RegisterMessage(&helpers.TestMessage{}) - logger, _ := helpers.NewFakeLogger() - - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() - - transport, err := amqp.NewTransport(cfg, resolver, logger) - if err != nil { - t.Skip("Skipping integration test - requires AMQP connection:", err) - - return - } - - msg := &helpers.TestMessage{ID: "123", Content: "test message"} - env := envelope.NewEnvelope(msg) - - t.Run("send method delegates to publisher", func(t *testing.T) { - sendCtx, sendCancel := context.WithTimeout(ctx, 2*time.Second) - defer sendCancel() - - sendErr := transport.Send(sendCtx, env) - assert.NoError(t, sendErr) - }) - - t.Run("receive method delegates to consumer", func(t *testing.T) { - handler := func(_ context.Context, _ api.Envelope) error { - return nil - } - - receiveCtx, receiveCancel := context.WithTimeout(ctx, 2*time.Second) - defer receiveCancel() - - receiveErr := transport.Receive(receiveCtx, handler) - assert.Error(t, receiveErr) - }) - - t.Run("retry method delegates to retry", func(t *testing.T) { - retryableTransport, ok := transport.(api.RetryableTransport) - if !ok { - t.Skip("Transport does not implement RetryableTransport") - - return - } - - retryCtx, retryCancel := context.WithTimeout(ctx, 2*time.Second) - defer retryCancel() - - retryErr := retryableTransport.Retry(retryCtx, env) - - assert.NoError(t, retryErr) - }) -} - -type MockTransport struct { - cfg amqp.TransportConfig - conn *mocks.MockConnectionAMQP -} - -func createMockTransport(cfg amqp.TransportConfig, conn *mocks.MockConnectionAMQP) *MockTransport { - return &MockTransport{ - cfg: cfg, - conn: conn, - } -} - -func (t *MockTransport) Setup() error { - ch, err := t.conn.Channel() - if err != nil { - return errors.New("failed to open channel: " + err.Error()) - } - defer ch.Close() - - err = ch.ExchangeDeclare( - t.cfg.Options.Exchange.Name, - t.cfg.Options.Exchange.Type, - t.cfg.Options.Exchange.Durable, - t.cfg.Options.Exchange.AutoDelete, - t.cfg.Options.Exchange.Internal, - false, - nil, - ) - if err != nil { - return errors.New("failed to declare exchange: " + err.Error()) - } - - for queueName, queueCfg := range t.cfg.Options.Queues { - _, err = ch.QueueDeclare( - queueName, - queueCfg.Durable, - queueCfg.AutoDelete, - queueCfg.Exclusive, - false, - nil, - ) - if err != nil { - return errors.New("declare queue: " + err.Error()) - } - - for _, bindingKey := range queueCfg.BindingKeys { - bindErr := ch.QueueBind( - queueName, - bindingKey, - t.cfg.Options.Exchange.Name, - false, - nil, - ) - if bindErr != nil { - return errors.New("bind queue: " + bindErr.Error()) - } - } - } - - return nil -} diff --git a/transport/chain.go b/transport/chain.go index 0726b9f..7ce3eb5 100644 --- a/transport/chain.go +++ b/transport/chain.go @@ -3,8 +3,11 @@ package transport import ( "fmt" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/core/config" + "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) type FactoryChain struct { @@ -15,10 +18,19 @@ func NewFactoryChain(factories ...api.TransportFactory) *FactoryChain { return &FactoryChain{factories: factories} } -func (c *FactoryChain) CreateTransport(name string, config config.TransportConfig) (api.Transport, error) { +func (c *FactoryChain) CreateTransport( + name string, + config config.TransportConfig, + sz api.Serializer, +) (api.Transport, error) { for _, factory := range c.factories { if factory.Supports(config.DSN) { - return factory.Create(name, config.DSN, config.Options) + rawOptions, errOptions := yaml.Marshal(config.Options) + if errOptions != nil { + return nil, fmt.Errorf("%s: marshal options map: %w", name, errOptions) + } + + return factory.Create(name, config.DSN, rawOptions, sz) } } diff --git a/transport/chain_test.go b/transport/chain_test.go index 7677188..032a0a8 100644 --- a/transport/chain_test.go +++ b/transport/chain_test.go @@ -6,10 +6,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" + + "github.com/gerfey/messenger/core/serializer" + "github.com/gerfey/messenger/transport" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/helpers" ) @@ -23,7 +27,7 @@ func (f *mockFactory) Supports(dsn string) bool { return dsn == f.supportedDSN } -func (f *mockFactory) Create(name, _ string, _ config.OptionsConfig) (api.Transport, error) { +func (f *mockFactory) Create(name, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { if f.createError != nil { return nil, f.createError } @@ -81,10 +85,13 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } - tr, err := chain.CreateTransport("test-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("test-transport", transportConfig, ser) require.NoError(t, err) assert.Same(t, expectedTransport, tr) @@ -103,10 +110,13 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } - tr, err := chain.CreateTransport("first-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("first-transport", transportConfig, ser) require.NoError(t, err) assert.Same(t, expectedTransport, tr) @@ -119,10 +129,13 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "unknown://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } - tr, err := chain.CreateTransport("unknown-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("unknown-transport", transportConfig, ser) require.Error(t, err) assert.Nil(t, tr) @@ -141,10 +154,13 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } - tr, err := chain.CreateTransport("error-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("error-transport", transportConfig, ser) require.Error(t, err) assert.Same(t, expectedError, err) @@ -156,10 +172,13 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "test://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } - tr, err := chain.CreateTransport("test-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("test-transport", transportConfig, ser) require.Error(t, err) assert.Nil(t, tr) @@ -179,10 +198,13 @@ func TestFactoryChain_CreateTransport(t *testing.T) { transportConfig := config.TransportConfig{ DSN: "redis://localhost", - Options: config.OptionsConfig{}, + Options: map[string]any{}, } - tr, err := chain.CreateTransport("second-transport", transportConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + tr, err := chain.CreateTransport("second-transport", transportConfig, ser) require.NoError(t, err) assert.Same(t, expectedTransport, tr) @@ -235,22 +257,25 @@ func TestFactoryChain_Integration(t *testing.T) { memoryConfig := config.TransportConfig{DSN: "memory://"} unknownConfig := config.TransportConfig{DSN: "unknown://localhost"} - amqpTransport, err := chain.CreateTransport("amqp", amqpConfig) + resolver := builder.NewResolver() + ser := serializer.NewSerializer(resolver) + + amqpTransport, err := chain.CreateTransport("amqp", amqpConfig, ser) require.NoError(t, err) require.NotNil(t, amqpTransport) assert.Equal(t, "amqp", amqpTransport.Name()) - redisTransport, err := chain.CreateTransport("redis", redisConfig) + redisTransport, err := chain.CreateTransport("redis", redisConfig, ser) require.NoError(t, err) require.NotNil(t, redisTransport) assert.Equal(t, "redis", redisTransport.Name()) - memoryTransport, err := chain.CreateTransport("memory", memoryConfig) + memoryTransport, err := chain.CreateTransport("memory", memoryConfig, ser) require.NoError(t, err) require.NotNil(t, memoryTransport) assert.Equal(t, "memory", memoryTransport.Name()) - unknownTransport, err := chain.CreateTransport("unknown", unknownConfig) + unknownTransport, err := chain.CreateTransport("unknown", unknownConfig, ser) require.Error(t, err) assert.Nil(t, unknownTransport) diff --git a/transport/inmemory/config.go b/transport/inmemory/config.go index a48147d..4d373f3 100644 --- a/transport/inmemory/config.go +++ b/transport/inmemory/config.go @@ -1,11 +1,7 @@ package inmemory -import ( - "github.com/gerfey/messenger/config" -) - type TransportConfig struct { Name string DSN string - Options config.OptionsConfig + Options any } diff --git a/transport/inmemory/factory.go b/transport/inmemory/factory.go index 600a227..3f11590 100644 --- a/transport/inmemory/factory.go +++ b/transport/inmemory/factory.go @@ -1,35 +1,21 @@ package inmemory import ( - "log/slog" "strings" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) -type TransportFactory struct { - logger *slog.Logger - resolver api.TypeResolver -} +type TransportFactory struct{} -func NewTransportFactory(logger *slog.Logger, resolver api.TypeResolver) api.TransportFactory { - return &TransportFactory{ - logger: logger, - resolver: resolver, - } +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} } func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "in-memory://") } -func (f *TransportFactory) Create(name string, dsn string, options config.OptionsConfig) (api.Transport, error) { - cfg := TransportConfig{ - Name: name, - DSN: dsn, - Options: options, - } - - return NewTransport(cfg), nil +func (f *TransportFactory) Create(name string, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { + return NewTransport(name), nil } diff --git a/transport/inmemory/factory_test.go b/transport/inmemory/factory_test.go index 3c1db29..90914d3 100644 --- a/transport/inmemory/factory_test.go +++ b/transport/inmemory/factory_test.go @@ -1,14 +1,15 @@ package inmemory_test import ( - "log/slog" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/core/serializer" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/inmemory" ) @@ -17,10 +18,7 @@ func TestNewTransportFactory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - - factory := inmemory.NewTransportFactory(logger, mockResolver) + factory := inmemory.NewTransportFactory() assert.NotNil(t, factory) assert.IsType(t, &inmemory.TransportFactory{}, factory) @@ -30,9 +28,7 @@ func TestTransportFactory_Supports(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() - mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := inmemory.NewTransportFactory(logger, mockResolver) + factory := inmemory.NewTransportFactory() testCases := []struct { name string @@ -73,15 +69,18 @@ func TestTransportFactory_Create(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockResolver := mocks.NewMockTypeResolver(ctrl) - factory := inmemory.NewTransportFactory(logger, mockResolver) + factory := inmemory.NewTransportFactory() name := "test-inmemory" dsn := "in-memory://test" - options := config.OptionsConfig{} + options := map[string]any{} + ser := serializer.NewSerializer(mockResolver) + + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) - transport, err := factory.Create(name, dsn, options) + transport, err := factory.Create(name, dsn, optionsBytes, ser) require.NoError(t, err) assert.NotNil(t, transport) diff --git a/transport/inmemory/transport.go b/transport/inmemory/transport.go index 861e2fe..5e5edc8 100644 --- a/transport/inmemory/transport.go +++ b/transport/inmemory/transport.go @@ -12,20 +12,20 @@ import ( const sleepDuration = 10 * time.Millisecond type Transport struct { - cfg TransportConfig + name string queue []api.Envelope lock sync.Mutex } -func NewTransport(cfg TransportConfig) api.Transport { +func NewTransport(name string) api.Transport { return &Transport{ - cfg: cfg, + name: name, queue: make([]api.Envelope, 0), } } func (t *Transport) Name() string { - return t.cfg.Name + return t.name } func (t *Transport) Send(_ context.Context, env api.Envelope) error { @@ -56,7 +56,7 @@ func (t *Transport) Receive(ctx context.Context, handler func(context.Context, a t.queue = t.queue[1:] t.lock.Unlock() - envWithReceivedStamp := env.WithStamp(stamps.ReceivedStamp{Transport: t.cfg.Name}) + envWithReceivedStamp := env.WithStamp(stamps.ReceivedStamp{Transport: t.name}) if err := handler(ctx, envWithReceivedStamp); err != nil { return err @@ -64,3 +64,12 @@ func (t *Transport) Receive(ctx context.Context, handler func(context.Context, a } } } + +func (t *Transport) Close() error { + t.lock.Lock() + defer t.lock.Unlock() + + t.queue = nil + + return nil +} diff --git a/transport/inmemory/transport_test.go b/transport/inmemory/transport_test.go index e1cf2f8..e7e5522 100644 --- a/transport/inmemory/transport_test.go +++ b/transport/inmemory/transport_test.go @@ -19,9 +19,7 @@ import ( func TestNewTransport(t *testing.T) { t.Run("create transport with config", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") require.NotNil(t, transport) assert.IsType(t, &inmemory.Transport{}, transport) @@ -29,27 +27,23 @@ func TestNewTransport(t *testing.T) { }) t.Run("create transport with empty config", func(t *testing.T) { - cfg := inmemory.TransportConfig{} - - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("in-memory") require.NotNil(t, transport) - assert.Empty(t, transport.Name()) + assert.Equal(t, "in-memory", transport.Name()) }) } func TestTransport_Name(t *testing.T) { t.Run("get transport name", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "my-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("my-transport") name := transport.Name() assert.Equal(t, "my-transport", name) }) t.Run("get empty transport name", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: ""} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("") name := transport.Name() assert.Empty(t, name) @@ -58,8 +52,7 @@ func TestTransport_Name(t *testing.T) { func TestTransport_Send(t *testing.T) { t.Run("send single message", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg).(*inmemory.Transport) + transport := inmemory.NewTransport("test-transport").(*inmemory.Transport) msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -70,8 +63,7 @@ func TestTransport_Send(t *testing.T) { }) t.Run("send multiple messages", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg).(*inmemory.Transport) + transport := inmemory.NewTransport("test-transport").(*inmemory.Transport) msg1 := &helpers.TestMessage{Content: "test1"} msg2 := &helpers.TestMessage{Content: "test2"} @@ -91,8 +83,7 @@ func TestTransport_Send(t *testing.T) { }) t.Run("send with cancelled context", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -107,8 +98,7 @@ func TestTransport_Send(t *testing.T) { func TestTransport_Receive(t *testing.T) { t.Run("receive single message", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -139,8 +129,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive multiple messages", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg1 := &helpers.TestMessage{Content: "test1"} msg2 := &helpers.TestMessage{Content: "test2"} @@ -170,8 +159,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive with handler error", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") msg := &helpers.TestMessage{Content: "test"} env := envelope.NewEnvelope(msg) @@ -192,8 +180,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive with cancelled context", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") ctx, cancel := context.WithCancel(t.Context()) cancel() @@ -208,8 +195,7 @@ func TestTransport_Receive(t *testing.T) { }) t.Run("receive with empty queue waits", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "test-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("test-transport") handler := func(_ context.Context, _ api.Envelope) error { return nil @@ -229,8 +215,7 @@ func TestTransport_Receive(t *testing.T) { func TestTransport_Integration(t *testing.T) { t.Run("full send and receive workflow", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "integration-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("integration-transport") messages := []*helpers.TestMessage{ {Content: "message1"}, @@ -269,8 +254,7 @@ func TestTransport_Integration(t *testing.T) { }) t.Run("concurrent send and receive", func(t *testing.T) { - cfg := inmemory.TransportConfig{Name: "concurrent-transport"} - transport := inmemory.NewTransport(cfg) + transport := inmemory.NewTransport("concurrent-transport") go func() { for range 5 { diff --git a/transport/kafka/benchmark_test.go b/transport/kafka/benchmark_test.go new file mode 100644 index 0000000..ebca1af --- /dev/null +++ b/transport/kafka/benchmark_test.go @@ -0,0 +1,121 @@ +package kafka_test + +import ( + "fmt" + "log/slog" + "sync" + "testing" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" +) + +const ( + benchmarkDSN = "kafka://localhost:29092/" +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func setupMessenger(b *testing.B) api.MessageBus { + b.Helper() + + logger := slog.New(slog.DiscardHandler) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "kafka": { + DSN: benchmarkDSN, + Serializer: "default.transport.serializer", + Options: map[string]any{ + "topics": []string{"benchmark-topic"}, + "group": "benchmark-group", + "producer": map[string]any{ + "async": true, + }, + }, + }, + }, + Routing: map[string]string{ + "*kafka_test.BenchmarkMessage": "kafka", + }, + } + + builderInstance := builder.NewBuilder(cfg, logger) + + builderInstance.RegisterMessage(&BenchmarkMessage{}) + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, size, false) + }) + } +} diff --git a/transport/kafka/config.go b/transport/kafka/config.go new file mode 100644 index 0000000..9579c74 --- /dev/null +++ b/transport/kafka/config.go @@ -0,0 +1,62 @@ +package kafka + +import "time" + +type TransportConfig struct { + Name string + DSN string + Options OptionsConfig +} + +type OptionsConfig struct { + Topics []string `yaml:"topics,omitempty"` + Group string `yaml:"group" default:"default-group"` + Producer ProducerOptionsConfig `yaml:"producer"` + Consumer ConsumerOptionsConfig `yaml:"consumer"` + Key KeyConfig `yaml:"key"` +} + +type ProducerOptionsConfig struct { + Async bool `yaml:"async" default:"false"` + AutoTopicCreation bool `yaml:"auto_topic_creation" default:"false"` + RequiredAcks int `yaml:"required_acks" default:"1"` // 0, 1, -1 (all) + BatchSize int `yaml:"batch_size" default:"256"` + BatchTimeout time.Duration `yaml:"batch_timeout" default:"5ms"` + WriteTimeout time.Duration `yaml:"write_timeout" default:"10s"` + ReadTimeout time.Duration `yaml:"read_timeout" default:"10s"` + Balancer string `yaml:"balancer" default:"round_robin"` // least_bytes, hash, round_robin +} + +type ConsumerOptionsConfig struct { + OffsetConfig OffsetConfig `yaml:"offset"` + Commit CommitConfig `yaml:"commit"` + Pool PoolConfig `yaml:"pool"` + Rebalance RebalanceConfig `yaml:"rebalance"` + SessionTimeout time.Duration `yaml:"session_timeout" default:"10s"` + HeartbeatInterval time.Duration `yaml:"heartbeat_interval" default:"2s"` +} + +type OffsetConfig struct { + Type string `yaml:"type" default:"latest"` // earliest, latest, specific + Value int64 `yaml:"value"` +} + +type CommitConfig struct { + Strategy string `yaml:"strategy" default:"batch"` // auto, manual, batch, deferred + Interval time.Duration `yaml:"interval" default:"500ms"` // only for batch + BatchSize int `yaml:"batch_size" default:"10"` // only for batch +} + +type PoolConfig struct { + Size int `yaml:"size" default:"3"` + MinSize int `yaml:"min_size" default:"2"` + MaxSize int `yaml:"max_size" default:"10"` +} + +type RebalanceConfig struct { + Strategy string `yaml:"strategy" default:"range"` // range, roundrobin +} + +type KeyConfig struct { + Strategy string `yaml:"strategy" default:"none"` // none, message_id +} diff --git a/transport/kafka/connection.go b/transport/kafka/connection.go new file mode 100644 index 0000000..577069c --- /dev/null +++ b/transport/kafka/connection.go @@ -0,0 +1,79 @@ +package kafka + +import ( + "context" + "fmt" + "time" + + "github.com/segmentio/kafka-go" +) + +const ( + connectionTimeout = 5 * time.Second +) + +type Connection struct { + brokers []string + dialer *kafka.Dialer +} + +func NewConnection(brokers []string) (ConnectionKafka, error) { + conn := &Connection{ + brokers: brokers, + dialer: &kafka.Dialer{ + Timeout: connectionTimeout, + DualStack: true, + }, + } + + if err := conn.check(connectionTimeout); err != nil { + return nil, err + } + + return conn, nil +} + +func (c *Connection) CreateReader(config kafka.ReaderConfig) *kafka.Reader { + config.Brokers = c.brokers + config.Dialer = c.dialer + + return kafka.NewReader(config) +} + +func (c *Connection) CreateWriter( + topic string, + opts ProducerOptionsConfig, + async bool, + balancer kafka.Balancer, +) *kafka.Writer { + return &kafka.Writer{ + Addr: kafka.TCP(c.brokers...), + Topic: topic, + RequiredAcks: kafka.RequiredAcks(opts.RequiredAcks), + Async: async, + AllowAutoTopicCreation: opts.AutoTopicCreation, + Balancer: balancer, + BatchSize: opts.BatchSize, + BatchTimeout: opts.BatchTimeout, + WriteTimeout: opts.WriteTimeout, + ReadTimeout: opts.ReadTimeout, + } +} + +func (c *Connection) check(timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + for _, broker := range c.brokers { + conn, connErr := c.dialer.DialContext(ctx, "tcp", broker) + if connErr != nil { + return fmt.Errorf("failed to connect to Kafka broker at '%s': %w", broker, connErr) + } + + if closeErr := conn.Close(); closeErr != nil { + return fmt.Errorf("failed to close connection to Kafka broker at '%s': %w", broker, closeErr) + } + } + + return nil +} diff --git a/transport/kafka/consumer.go b/transport/kafka/consumer.go new file mode 100644 index 0000000..1789e31 --- /dev/null +++ b/transport/kafka/consumer.go @@ -0,0 +1,322 @@ +package kafka + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/segmentio/kafka-go" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" +) + +const ( + minBytes = 10e3 // 10KB + maxBytes = 10e6 // 10MB + rebalanceTimeout = 5 * time.Second + defaultPoolSize = 10 + readLagInterval = -1 + errorBackoffDelay = 100 * time.Millisecond +) + +type Consumer struct { + config TransportConfig + serializer api.Serializer + connection ConnectionKafka + readers []*kafka.Reader + wg sync.WaitGroup + batchMutex sync.Mutex + batchMessages []kafka.Message + deferredCommits sync.Map +} + +type messageWithReader struct { + message kafka.Message + reader *kafka.Reader +} + +func NewConsumer(config TransportConfig, connection ConnectionKafka, serializer api.Serializer) (api.Consumer, error) { + return &Consumer{ + config: config, + connection: connection, + serializer: serializer, + readers: make([]*kafka.Reader, 0), + }, nil +} + +func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + for _, topic := range c.config.Options.Topics { + readerConfig := kafka.ReaderConfig{ + GroupID: c.config.Options.Group, + Topic: topic, + CommitInterval: c.config.Options.Consumer.Commit.Interval, + MinBytes: minBytes, + MaxBytes: maxBytes, + ReadLagInterval: readLagInterval, + SessionTimeout: c.config.Options.Consumer.SessionTimeout, + RebalanceTimeout: rebalanceTimeout, + HeartbeatInterval: c.config.Options.Consumer.HeartbeatInterval, + MaxWait: time.Second, + } + + switch c.config.Options.Consumer.Rebalance.Strategy { + case "range": + readerConfig.GroupBalancers = []kafka.GroupBalancer{kafka.RangeGroupBalancer{}} + case "roundrobin": + readerConfig.GroupBalancers = []kafka.GroupBalancer{kafka.RoundRobinGroupBalancer{}} + } + + c.configureOffset(&readerConfig) + reader := c.connection.CreateReader(readerConfig) + c.readers = append(c.readers, reader) + } + + jobs := make(chan job) + c.startWorkerPool(ctx, jobs, handler) + + for _, reader := range c.readers { + c.wg.Add(1) + go func(r *kafka.Reader) { + defer c.wg.Done() + c.fetchMessages(ctx, r, jobs) + }(reader) + } + + if c.config.Options.Consumer.Commit.Strategy == "batch" { + go c.startBatchCommitter(ctx) + } + + <-ctx.Done() + + close(jobs) + c.wg.Wait() + + return ctx.Err() +} + +func (c *Consumer) Close() error { + for _, reader := range c.readers { + err := reader.Close() + if err != nil { + return err + } + } + + return nil +} + +func (c *Consumer) configureOffset(config *kafka.ReaderConfig) { + switch c.config.Options.Consumer.OffsetConfig.Type { + case "earliest": + config.StartOffset = kafka.FirstOffset + case "specific": + config.StartOffset = c.config.Options.Consumer.OffsetConfig.Value + default: + config.StartOffset = kafka.LastOffset + } +} + +func (c *Consumer) startWorkerPool( + ctx context.Context, + jobs chan job, + handler func(context.Context, api.Envelope) error, +) { + poolSize := c.config.Options.Consumer.Pool.Size + if poolSize <= 0 { + poolSize = defaultPoolSize + } + + for range poolSize { + c.wg.Add(1) + go c.startWorker(ctx, jobs, handler) + } +} + +func (c *Consumer) startWorker(ctx context.Context, jobs chan job, handler func(context.Context, api.Envelope) error) { + defer c.wg.Done() + + for { + select { + case <-ctx.Done(): + return + case j, ok := <-jobs: + if !ok { + return + } + c.handleMessage(ctx, j.r, j.msg, handler) + } + } +} + +func (c *Consumer) fetchMessages(ctx context.Context, r *kafka.Reader, jobs chan job) { + for { + select { + case <-ctx.Done(): + return + default: + msg, err := r.FetchMessage(ctx) + if err != nil { + if ctx.Err() != nil { + return + } + + time.Sleep(errorBackoffDelay) + + continue + } + + select { + case <-ctx.Done(): + return + case jobs <- job{r: r, msg: msg}: + } + } + } +} + +func (c *Consumer) handleMessage( + ctx context.Context, + r *kafka.Reader, + msg kafka.Message, + handler func(context.Context, api.Envelope) error, +) { + env, err := c.serializer.Unmarshal(msg.Value, c.headerMap(msg.Headers)) + if err != nil { + c.commitMessage(ctx, r, msg) + + return + } + + env = env.WithStamp(stamps.ReceivedStamp{Transport: c.config.Name}) + + if handlerErr := handler(ctx, env); handlerErr != nil { + c.commitMessage(ctx, r, msg) + + return + } + + c.commitMessage(ctx, r, msg) +} + +func (c *Consumer) commitMessage(ctx context.Context, r *kafka.Reader, msg kafka.Message) { + switch c.config.Options.Consumer.Commit.Strategy { + case "auto": + _ = r.CommitMessages(ctx, msg) + case "manual": + case "batch": + c.batchMutex.Lock() + c.batchMessages = append(c.batchMessages, msg) + + msgKey := fmt.Sprintf("%s-%d-%d", msg.Topic, msg.Partition, msg.Offset) + c.deferredCommits.Store(msgKey, messageWithReader{message: msg, reader: r}) + + if len(c.batchMessages) >= c.config.Options.Consumer.Commit.BatchSize { + c.batchMutex.Unlock() + c.commitBatch(ctx) + } else { + c.batchMutex.Unlock() + } + case "deferred": + msgKey := fmt.Sprintf("%s-%d-%d", msg.Topic, msg.Partition, msg.Offset) + c.deferredCommits.Store(msgKey, messageWithReader{message: msg, reader: r}) + } +} + +func (c *Consumer) startBatchCommitter(ctx context.Context) { + ticker := time.NewTicker(c.config.Options.Consumer.Commit.Interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.commitBatch(ctx) + } + } +} + +func (c *Consumer) commitBatch(ctx context.Context) { + c.batchMutex.Lock() + defer c.batchMutex.Unlock() + + readerMessages := c.groupMessagesByReader() + + for reader, messages := range readerMessages { + if len(messages) == 0 { + continue + } + + partitionOffsets := c.findMaxOffsetPerPartition(messages) + + c.commitMessagesAndCleanup(ctx, reader, messages, partitionOffsets) + } + + c.batchMessages = c.batchMessages[:0] +} + +func (c *Consumer) groupMessagesByReader() map[*kafka.Reader][]kafka.Message { + readerMessages := make(map[*kafka.Reader][]kafka.Message) + + c.deferredCommits.Range(func(_, value any) bool { + if msgWithReader, ok := value.(messageWithReader); ok { + readerMessages[msgWithReader.reader] = append(readerMessages[msgWithReader.reader], msgWithReader.message) + } + + return true + }) + + return readerMessages +} + +func (c *Consumer) findMaxOffsetPerPartition(messages []kafka.Message) map[int]kafka.Message { + partitionOffsets := make(map[int]kafka.Message) + + for _, msg := range messages { + currentMax, exists := partitionOffsets[msg.Partition] + if !exists || currentMax.Offset < msg.Offset { + partitionOffsets[msg.Partition] = msg + } + } + + return partitionOffsets +} + +func (c *Consumer) commitMessagesAndCleanup( + ctx context.Context, + reader *kafka.Reader, + messages []kafka.Message, + partitionOffsets map[int]kafka.Message, +) { + for _, msg := range partitionOffsets { + if err := reader.CommitMessages(ctx, msg); err == nil { + c.cleanupCommittedMessages(messages, msg) + } + } +} + +func (c *Consumer) cleanupCommittedMessages(messages []kafka.Message, committedMsg kafka.Message) { + for _, commitedMsg := range messages { + if commitedMsg.Partition == committedMsg.Partition && commitedMsg.Offset <= committedMsg.Offset { + c.deferredCommits.Delete( + fmt.Sprintf("%s-%d-%d", commitedMsg.Topic, commitedMsg.Partition, commitedMsg.Offset), + ) + } + } +} + +type job struct { + r *kafka.Reader + msg kafka.Message +} + +func (c *Consumer) headerMap(headers []kafka.Header) map[string]string { + m := make(map[string]string) + for _, h := range headers { + m[h.Key] = string(h.Value) + } + + return m +} diff --git a/transport/kafka/factory.go b/transport/kafka/factory.go new file mode 100644 index 0000000..732d6a9 --- /dev/null +++ b/transport/kafka/factory.go @@ -0,0 +1,40 @@ +package kafka + +import ( + "fmt" + "strings" + + "github.com/creasty/defaults" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/api" +) + +type TransportFactory struct{} + +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} +} + +func (t *TransportFactory) Supports(dsn string) bool { + return strings.HasPrefix(dsn, "kafka://") +} + +func (t *TransportFactory) Create(name string, dsn string, options []byte, ser api.Serializer) (api.Transport, error) { + var optsConfig OptionsConfig + if err := defaults.Set(&optsConfig); err != nil { + return nil, fmt.Errorf("set defaults: %w", err) + } + + if err := yaml.Unmarshal(options, &optsConfig); err != nil { + return nil, fmt.Errorf("unmarshal options: %w", err) + } + + tCfg := TransportConfig{ + Name: name, + DSN: dsn, + Options: optsConfig, + } + + return NewTransport(tCfg, ser) +} diff --git a/transport/kafka/factory_test.go b/transport/kafka/factory_test.go new file mode 100644 index 0000000..f5be521 --- /dev/null +++ b/transport/kafka/factory_test.go @@ -0,0 +1,95 @@ +package kafka_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/core/serializer" + + "github.com/gerfey/messenger/tests/mocks" + "github.com/gerfey/messenger/transport/kafka" +) + +func TestNewTransportFactory(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + factory := kafka.NewTransportFactory() + + assert.NotNil(t, factory) + assert.IsType(t, &kafka.TransportFactory{}, factory) +} + +func TestTransportFactory_Supports(t *testing.T) { + testCases := []struct { + name string + dsn string + expected bool + }{ + { + name: "supports kafka dsn", + dsn: "kafka://localhost:9092", + expected: true, + }, + { + name: "does not support amqp dsn", + dsn: "amqp://guest:guest@localhost:5672/", + expected: false, + }, + { + name: "does not support in-memory dsn", + dsn: "in-memory://", + expected: false, + }, + { + name: "does not support sync dsn", + dsn: "sync://", + expected: false, + }, + { + name: "does not support empty dsn", + dsn: "", + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + factory := kafka.NewTransportFactory() + + result := factory.Supports(tc.dsn) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestTransportFactory_Create(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockResolver := mocks.NewMockTypeResolver(ctrl) + factory := kafka.NewTransportFactory() + + name := "test-kafka" + dsn := "kafka://non-existent-host:9092" + options := kafka.OptionsConfig{ + Topics: []string{"test-topic"}, + Group: "test-group", + } + ser := serializer.NewSerializer(mockResolver) + + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) + + transport, err := factory.Create(name, dsn, optionsBytes, ser) + + require.Error(t, err) + assert.Nil(t, transport) +} diff --git a/transport/kafka/producer.go b/transport/kafka/producer.go new file mode 100644 index 0000000..f68aefc --- /dev/null +++ b/transport/kafka/producer.go @@ -0,0 +1,133 @@ +package kafka + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/segmentio/kafka-go" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" +) + +type Producer struct { + config TransportConfig + serializer api.Serializer + connection ConnectionKafka + writers map[string]*kafka.Writer + mu sync.RWMutex +} + +func NewProducer(config TransportConfig, connection ConnectionKafka, serializer api.Serializer) (api.Producer, error) { + p := &Producer{ + config: config, + connection: connection, + serializer: serializer, + writers: make(map[string]*kafka.Writer), + } + + if len(config.Options.Topics) == 0 { + return nil, errors.New("no topics configured for kafka transport") + } + + var balancer kafka.Balancer = &kafka.LeastBytes{} + switch config.Options.Producer.Balancer { + case "hash": + balancer = &kafka.Hash{} + case "round_robin": + balancer = &kafka.RoundRobin{} + case "least_bytes": + balancer = &kafka.LeastBytes{} + } + + if config.Options.Key.Strategy != "none" { + balancer = &kafka.Hash{} + } + + for _, topic := range config.Options.Topics { + writer := connection.CreateWriter( + topic, + config.Options.Producer, + config.Options.Producer.Async, + balancer, + ) + + p.writers[topic] = writer + } + + return p, nil +} + +func (p *Producer) Send(ctx context.Context, env api.Envelope) error { + payload, headers, err := p.serializer.Marshal(env) + if err != nil { + return fmt.Errorf("serializer envelope failed: %w", err) + } + + kHeaders := make([]kafka.Header, 0, len(headers)) + for k, v := range headers { + kHeaders = append(kHeaders, kafka.Header{Key: k, Value: []byte(v)}) + } + + msg := kafka.Message{ + Headers: kHeaders, + Value: payload, + Time: time.Now(), + } + + key, keyErr := p.extractMessageKey(env) + if keyErr == nil && len(key) > 0 { + msg.Key = key + } + + p.mu.RLock() + defer p.mu.RUnlock() + + for _, topic := range p.config.Options.Topics { + writer, exists := p.writers[topic] + if !exists { + return fmt.Errorf("writer for topic %s not found", topic) + } + + if writeErr := writer.WriteMessages(ctx, msg); writeErr != nil { + return fmt.Errorf("producer failed to write messages to topic %s: %w", topic, writeErr) + } + } + + return nil +} + +func (p *Producer) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + + var errs []error + for topic, writer := range p.writers { + if err := writer.Close(); err != nil { + errs = append(errs, fmt.Errorf("failed to close writer for topic %s: %w", topic, err)) + } + } + + if len(errs) > 0 { + return fmt.Errorf("errors closing writers: %v", errs) + } + + return nil +} + +func (p *Producer) extractMessageKey(env api.Envelope) ([]byte, error) { + if p.config.Options.Key.Strategy != "message_id" { + return nil, nil + } + + for _, s := range env.Stamps() { + if msgIDStamp, ok := s.(stamps.MessageIDStamp); ok { + return []byte(msgIDStamp.MessageID), nil + } + } + + return nil, errors.New("message_id stamp not found") +} diff --git a/transport/kafka/transport.go b/transport/kafka/transport.go new file mode 100644 index 0000000..7b52810 --- /dev/null +++ b/transport/kafka/transport.go @@ -0,0 +1,75 @@ +package kafka + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/segmentio/kafka-go" + + "github.com/gerfey/messenger/api" +) + +type ConnectionKafka interface { + CreateReader(kafka.ReaderConfig) *kafka.Reader + CreateWriter(string, ProducerOptionsConfig, bool, kafka.Balancer) *kafka.Writer +} + +type Transport struct { + cfg TransportConfig + producer api.Producer + consumer api.Consumer + connection ConnectionKafka +} + +func NewTransport(cfg TransportConfig, serializer api.Serializer) (api.Transport, error) { + u, err := url.Parse(cfg.DSN) + if err != nil { + return nil, fmt.Errorf("failed to parse dsn: %w", err) + } + + brokers := strings.Split(u.Host, ",") + + connection, errConnection := NewConnection(brokers) + if errConnection != nil { + return nil, fmt.Errorf("failed to create connection: %w", errConnection) + } + + producer, errProducer := NewProducer(cfg, connection, serializer) + if errProducer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errProducer) + } + + consumer, errConsumer := NewConsumer(cfg, connection, serializer) + if errConsumer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errConsumer) + } + + return &Transport{ + cfg: cfg, + producer: producer, + consumer: consumer, + connection: connection, + }, nil +} + +func (t *Transport) Name() string { + return t.cfg.Name +} + +func (t *Transport) Send(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} + +func (t *Transport) Receive(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + return t.consumer.Consume(ctx, handler) +} + +func (t *Transport) Close() error { + return t.producer.Close() +} + +func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} diff --git a/transport/manager.go b/transport/manager.go index a7b2785..ec1274a 100644 --- a/transport/manager.go +++ b/transport/manager.go @@ -53,6 +53,14 @@ func (m *Manager) Start(ctx context.Context, consumeOnly []string) { m.running = true for _, t := range m.transports { + if s, ok := t.(api.SetupableTransport); ok { + if err := s.Setup(ctx); err != nil { + m.logger.ErrorContext(ctx, "failed to setup transport", "name", t.Name(), "error", err) + + continue + } + } + if !m.stringInSlice(t.Name(), consumeOnly) { continue } @@ -65,6 +73,12 @@ func (m *Manager) Stop() { m.running = false m.mu.Unlock() + for _, t := range m.transports { + if errClose := t.Close(); errClose != nil { + m.logger.Error("failed to close transport", "name", t.Name(), "error", errClose) + } + } + m.wg.Wait() } @@ -129,7 +143,7 @@ func (m *Manager) receiveTransport(ctx context.Context, t api.Transport) { }) if err != nil { - m.logger.Error("receive error", "error", err) + m.logger.ErrorContext(ctx, "receive error", "transport", t.Name(), "error", err) } }(t) } diff --git a/transport/redis/benchmark_test.go b/transport/redis/benchmark_test.go new file mode 100644 index 0000000..7c18d5e --- /dev/null +++ b/transport/redis/benchmark_test.go @@ -0,0 +1,120 @@ +package redis_test + +import ( + "fmt" + "log/slog" + "sync" + "testing" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" +) + +const ( + benchmarkDSN = "redis://localhost:6379/0" +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func setupMessenger(b *testing.B) api.MessageBus { + b.Helper() + + logger := slog.New(slog.DiscardHandler) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "redis": { + DSN: benchmarkDSN, + Serializer: "default.transport.serializer", + Options: map[string]any{ + "auto_setup": true, + "stream": "benchmark_stream", + "group": "benchmark_group", + "consumer": "benchmark_consumer", + }, + }, + }, + Routing: map[string]string{ + "*redis_test.BenchmarkMessage": "redis", + }, + } + + builderInstance := builder.NewBuilder(cfg, logger) + + builderInstance.RegisterMessage(&BenchmarkMessage{}) + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, size, false) + }) + } +} diff --git a/transport/redis/config.go b/transport/redis/config.go new file mode 100644 index 0000000..2725c18 --- /dev/null +++ b/transport/redis/config.go @@ -0,0 +1,14 @@ +package redis + +type TransportConfig struct { + Name string + DSN string + Options OptionsConfig +} + +type OptionsConfig struct { + AutoSetup bool `yaml:"auto_setup" default:"true"` + Stream string `yaml:"stream" default:"messages"` + Group string `yaml:"group" default:"default"` + Consumer string `yaml:"consumer" default:"consumer"` +} diff --git a/transport/redis/connection.go b/transport/redis/connection.go new file mode 100644 index 0000000..71b97c4 --- /dev/null +++ b/transport/redis/connection.go @@ -0,0 +1,48 @@ +package redis + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/redis/go-redis/v9" +) + +const connectionTimeout = 5 * time.Second + +type Connection struct { + client *redis.Client +} + +func NewConnection(dsn string) (*Connection, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, fmt.Errorf("failed to parse dsn: %w", err) + } + + opts := &redis.Options{ + Addr: u.Host, + } + + if u.User != nil { + if password, hasPassword := u.User.Password(); hasPassword { + opts.Password = password + } + } + + client := redis.NewClient(opts) + + ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) + defer cancel() + + if errPing := client.Ping(ctx).Err(); errPing != nil { + return nil, fmt.Errorf("ping failed: %w", errPing) + } + + return &Connection{client: client}, nil +} + +func (c *Connection) Client() *redis.Client { + return c.client +} diff --git a/transport/redis/consumer.go b/transport/redis/consumer.go new file mode 100644 index 0000000..de9505a --- /dev/null +++ b/transport/redis/consumer.go @@ -0,0 +1,128 @@ +package redis + +import ( + "context" + "errors" + "strings" + "sync" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/stamps" +) + +const ( + defaultBatchSize = 10 +) + +type Consumer struct { + config TransportConfig + serializer api.Serializer + connection ConnectionRedis + wg sync.WaitGroup +} + +func NewConsumer(config TransportConfig, serializer api.Serializer, connection ConnectionRedis) (api.Consumer, error) { + return &Consumer{ + config: config, + serializer: serializer, + connection: connection, + }, nil +} + +func (c *Consumer) Consume(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + group := c.config.Options.Group + stream := c.config.Options.Stream + + _ = c.connection.Client().XGroupCreateMkStream(ctx, stream, group, "$") + + c.wg.Add(1) + go func() { + defer c.wg.Done() + c.consumeLoop(ctx, handler) + }() + + <-ctx.Done() + + c.wg.Wait() + + return ctx.Err() +} + +func (c *Consumer) Close() error { + return nil +} + +func (c *Consumer) consumeLoop(ctx context.Context, handler func(context.Context, api.Envelope) error) { + rdb := c.connection.Client() + stream := c.config.Options.Stream + group := c.config.Options.Group + consumer := c.config.Options.Consumer + + for { + select { + case <-ctx.Done(): + return + default: + streams, err := rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: group, + Consumer: consumer, + Streams: []string{stream, ">"}, + Count: defaultBatchSize, + Block: time.Second, + }).Result() + + if err != nil && !errors.Is(err, redis.Nil) { + continue + } + + for _, s := range streams { + for _, msg := range s.Messages { + c.handleMessage(ctx, msg, handler) + } + } + } + } +} + +func (c *Consumer) handleMessage( + ctx context.Context, + msg redis.XMessage, + handler func(context.Context, api.Envelope) error, +) { + bodyRaw, ok := msg.Values["body"] + if !ok { + return + } + + bodyBytes, ok := bodyRaw.(string) + if !ok { + return + } + + headers := make(map[string]string) + for k, v := range msg.Values { + if strings.HasPrefix(k, "header_") { + if str, isString := v.(string); isString { + headers[strings.TrimPrefix(k, "header_")] = str + } + } + } + + env, errUnmarshal := c.serializer.Unmarshal([]byte(bodyBytes), headers) + if errUnmarshal != nil { + return + } + + env = env.WithStamp(stamps.ReceivedStamp{Transport: c.config.Name}) + + if errHandler := handler(ctx, env); errHandler != nil { + return + } + + if err := c.connection.Client().XAck(ctx, c.config.Options.Stream, c.config.Options.Group, msg.ID).Err(); err != nil { + return + } +} diff --git a/transport/redis/factory.go b/transport/redis/factory.go new file mode 100644 index 0000000..c9c456f --- /dev/null +++ b/transport/redis/factory.go @@ -0,0 +1,45 @@ +package redis + +import ( + "fmt" + "strings" + + "github.com/creasty/defaults" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/api" +) + +type TransportFactory struct{} + +func NewTransportFactory() api.TransportFactory { + return &TransportFactory{} +} + +func (t *TransportFactory) Supports(dsn string) bool { + return strings.HasPrefix(dsn, "redis://") +} + +func (t *TransportFactory) Create( + name string, + dsn string, + options []byte, + serializer api.Serializer, +) (api.Transport, error) { + var optsConfig OptionsConfig + if err := defaults.Set(&optsConfig); err != nil { + return nil, fmt.Errorf("set defaults: %w", err) + } + + if err := yaml.Unmarshal(options, &optsConfig); err != nil { + return nil, fmt.Errorf("unmarshal options: %w", err) + } + + tCfg := TransportConfig{ + Name: name, + DSN: dsn, + Options: optsConfig, + } + + return NewTransport(tCfg, serializer) +} diff --git a/transport/redis/producer.go b/transport/redis/producer.go new file mode 100644 index 0000000..5440eaf --- /dev/null +++ b/transport/redis/producer.go @@ -0,0 +1,74 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "regexp" + + "github.com/redis/go-redis/v9" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/envelope" + "github.com/gerfey/messenger/core/stamps" +) + +type Producer struct { + config TransportConfig + serializer api.Serializer + connection ConnectionRedis +} + +func NewProducer(config TransportConfig, serializer api.Serializer, connection ConnectionRedis) (api.Producer, error) { + return &Producer{ + config: config, + serializer: serializer, + connection: connection, + }, nil +} + +func (p *Producer) Send(ctx context.Context, env api.Envelope) error { + payload, headers, err := p.serializer.Marshal(env) + if err != nil { + return fmt.Errorf("redis: marshal envelope failed: %w", err) + } + + data := map[string]any{ + "body": payload, + } + + for k, v := range headers { + data["header_"+k] = v + } + + stream := p.config.Options.Stream + if stream == "" { + return errors.New("redis: stream name is not configured") + } + + id := "*" + if stamp, ok := envelope.LastStampOf[stamps.MessageIDStamp](env); ok { + if p.isValidRedisStreamID(stamp.MessageID) { + id = stamp.MessageID + } + } + + _, err = p.connection.Client().XAdd(ctx, &redis.XAddArgs{ + ID: id, + Stream: stream, + Values: data, + }).Result() + if err != nil { + return fmt.Errorf("redis: XADD failed: %w", err) + } + + return nil +} + +func (p *Producer) Close() error { + return nil +} + +func (p *Producer) isValidRedisStreamID(id string) bool { + return regexp.MustCompile(`^\d+-\d+$`).MatchString(id) +} diff --git a/transport/redis/transport.go b/transport/redis/transport.go new file mode 100644 index 0000000..942f676 --- /dev/null +++ b/transport/redis/transport.go @@ -0,0 +1,82 @@ +package redis + +import ( + "context" + "fmt" + "strings" + + "github.com/redis/go-redis/v9" + + "github.com/gerfey/messenger/api" +) + +type ConnectionRedis interface { + Client() *redis.Client +} + +type Transport struct { + cfg TransportConfig + producer api.Producer + consumer api.Consumer + connection ConnectionRedis +} + +func NewTransport(cfg TransportConfig, serializer api.Serializer) (api.Transport, error) { + connection, errConnection := NewConnection(cfg.DSN) + if errConnection != nil { + return nil, fmt.Errorf("failed to create connection: %w", errConnection) + } + + producer, errProducer := NewProducer(cfg, serializer, connection) + if errProducer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errProducer) + } + + consumer, errConsumer := NewConsumer(cfg, serializer, connection) + if errConsumer != nil { + return nil, fmt.Errorf("failed to create producer: %w", errConsumer) + } + + return &Transport{ + cfg: cfg, + producer: producer, + consumer: consumer, + connection: connection, + }, nil +} + +func (t *Transport) Name() string { + return t.cfg.Name +} + +func (t *Transport) Send(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} + +func (t *Transport) Receive(ctx context.Context, handler func(context.Context, api.Envelope) error) error { + return t.consumer.Consume(ctx, handler) +} + +func (t *Transport) Retry(ctx context.Context, env api.Envelope) error { + return t.producer.Send(ctx, env) +} + +func (t *Transport) Setup(ctx context.Context) error { + if !t.cfg.Options.AutoSetup { + return nil + } + + stream := t.cfg.Options.Stream + group := t.cfg.Options.Group + + _, err := t.connection.Client().XGroupCreateMkStream(ctx, stream, group, "$").Result() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return fmt.Errorf("failed to create consumer group: %w", err) + } + + return nil +} + +func (t *Transport) Close() error { + return nil +} diff --git a/transport/sync/benchmark_test.go b/transport/sync/benchmark_test.go new file mode 100644 index 0000000..dc83195 --- /dev/null +++ b/transport/sync/benchmark_test.go @@ -0,0 +1,114 @@ +package sync_test + +import ( + "fmt" + "log/slog" + "sync" + "testing" + + "github.com/gerfey/messenger/api" + "github.com/gerfey/messenger/core/builder" + "github.com/gerfey/messenger/core/config" +) + +const ( + benchmarkDSN = "sync://" +) + +type BenchmarkMessage struct { + ID string + Content string + Data []byte +} + +func setupMessenger(b *testing.B) api.MessageBus { + b.Helper() + + logger := slog.New(slog.DiscardHandler) + + cfg := &config.MessengerConfig{ + DefaultBus: "default", + Buses: map[string]config.BusConfig{ + "default": {}, + }, + Transports: map[string]config.TransportConfig{ + "sync": { + DSN: benchmarkDSN, + Serializer: "default.transport.serializer", + }, + }, + Routing: map[string]string{ + "*sync_test.BenchmarkMessage": "sync", + }, + } + + builderInstance := builder.NewBuilder(cfg, logger) + + builderInstance.RegisterMessage(&BenchmarkMessage{}) + + messenger, err := builderInstance.Build() + if err != nil { + b.Fatalf("Build messenger failed: %v", err) + } + + bus, err := messenger.GetDefaultBus() + if err != nil { + b.Fatalf("Get default bus failed: %v", err) + } + + return bus +} + +func dispatchMessages(b *testing.B, bus api.MessageBus, size int, parallel bool) { + ctx := b.Context() + b.ResetTimer() + b.ReportAllocs() + + if parallel { + concurrency := 10 + var wg sync.WaitGroup + messagesPerWorker := b.N / concurrency + for w := range concurrency { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := range messagesPerWorker { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("worker-%d-msg-%d", id, i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + }(w) + } + wg.Wait() + } else { + for i := range b.N { + bus.Dispatch(ctx, &BenchmarkMessage{ + ID: fmt.Sprintf("msg-%d", i), + Content: "benchmark content", + Data: make([]byte, size), + }) + } + } +} + +func BenchmarkSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, false) +} + +func BenchmarkConcurrentSend(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, 100, true) +} + +func BenchmarkMessageSizes(b *testing.B) { + sizes := []int{100, 1024, 10240, 102400} + for _, size := range sizes { + b.Run(fmt.Sprintf("Size_%dB", size), func(b *testing.B) { + bus := setupMessenger(b) + dispatchMessages(b, bus, size, false) + }) + } +} diff --git a/transport/sync/factory.go b/transport/sync/factory.go index 822e4df..9ae0c6f 100644 --- a/transport/sync/factory.go +++ b/transport/sync/factory.go @@ -1,29 +1,25 @@ package sync import ( - "log/slog" "strings" "github.com/gerfey/messenger/api" - "github.com/gerfey/messenger/config" ) -type Factory struct { - logger *slog.Logger +type TransportFactory struct { locator api.BusLocator } -func NewTransportFactory(logger *slog.Logger, locator api.BusLocator) api.TransportFactory { - return &Factory{ - logger: logger, +func NewTransportFactory(locator api.BusLocator) api.TransportFactory { + return &TransportFactory{ locator: locator, } } -func (f *Factory) Supports(dsn string) bool { +func (f *TransportFactory) Supports(dsn string) bool { return strings.HasPrefix(dsn, "sync://") } -func (f *Factory) Create(_ string, _ string, _ config.OptionsConfig) (api.Transport, error) { +func (f *TransportFactory) Create(_ string, _ string, _ []byte, _ api.Serializer) (api.Transport, error) { return NewTransport(f.locator), nil } diff --git a/transport/sync/factory_test.go b/transport/sync/factory_test.go index 9e1e6f7..09f1255 100644 --- a/transport/sync/factory_test.go +++ b/transport/sync/factory_test.go @@ -1,14 +1,15 @@ package sync_test import ( - "log/slog" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "gopkg.in/yaml.v3" + + "github.com/gerfey/messenger/core/serializer" - "github.com/gerfey/messenger/config" "github.com/gerfey/messenger/tests/mocks" "github.com/gerfey/messenger/transport/sync" ) @@ -17,22 +18,20 @@ func TestNewTransportFactory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockLocator := mocks.NewMockBusLocator(ctrl) - factory := sync.NewTransportFactory(logger, mockLocator) + factory := sync.NewTransportFactory(mockLocator) assert.NotNil(t, factory) - assert.IsType(t, &sync.Factory{}, factory) + assert.IsType(t, &sync.TransportFactory{}, factory) } func TestFactory_Supports(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockLocator := mocks.NewMockBusLocator(ctrl) - factory := sync.NewTransportFactory(logger, mockLocator) + factory := sync.NewTransportFactory(mockLocator) testCases := []struct { name string @@ -73,15 +72,19 @@ func TestFactory_Create(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logger := slog.Default() mockLocator := mocks.NewMockBusLocator(ctrl) - factory := sync.NewTransportFactory(logger, mockLocator) + factory := sync.NewTransportFactory(mockLocator) name := "test-sync" dsn := "sync://" - options := config.OptionsConfig{} + options := map[string]any{} + mockResolver := mocks.NewMockTypeResolver(ctrl) + ser := serializer.NewSerializer(mockResolver) + + optionsBytes, err := yaml.Marshal(options) + require.NoError(t, err) - transport, err := factory.Create(name, dsn, options) + transport, err := factory.Create(name, dsn, optionsBytes, ser) require.NoError(t, err) assert.NotNil(t, transport) diff --git a/transport/sync/transport.go b/transport/sync/transport.go index e0de523..0205e23 100644 --- a/transport/sync/transport.go +++ b/transport/sync/transport.go @@ -17,7 +17,7 @@ func NewTransport(locator api.BusLocator) api.Transport { return &Transport{locator: locator} } -func (t *Transport) Send(_ context.Context, env api.Envelope) error { +func (t *Transport) Send(ctx context.Context, env api.Envelope) error { busNameStump, ok := envelope.LastStampOf[stamps.BusNameStamp](env) if !ok { return errors.New("no BusNameStamp found in envelope") @@ -30,7 +30,7 @@ func (t *Transport) Send(_ context.Context, env api.Envelope) error { env = env.WithStamp(stamps.ReceivedStamp{Transport: t.Name()}) - _, err := messageBus.Dispatch(context.Background(), env) + _, err := messageBus.Dispatch(ctx, env) if err != nil { return err } @@ -45,3 +45,7 @@ func (t *Transport) Receive(_ context.Context, _ func(context.Context, api.Envel func (t *Transport) Name() string { return "sync" } + +func (t *Transport) Close() error { + return nil +}