Skip to content

Use one shared OpenSearch and Elasticsearch container for all projects #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
peterjaap wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Use one shared OpenSearch and Elasticsearch container for all projects #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/magebox/check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ func runCheck(cmd *cobra.Command, args []string) error {
// Check project-specific services
if cfg != nil {
if cfg.Services.HasOpenSearch() {
osPort := docker.GetOpenSearchPort(cfg.Services.OpenSearch.Version)
osPort := docker.GetOpenSearchPort()
if dockerCtrl.IsServiceRunning("opensearch") {
results = append(results, checkResult{
name: "OpenSearch",
Expand Down
6 changes: 3 additions & 3 deletions cmd/magebox/new.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -807,15 +807,15 @@ commands:

// Add search engine config
if searchEngine == "opensearch" {
searchPort := docker.GetOpenSearchPort(searchVersion)
searchPort := docker.GetOpenSearchPort()
installCmd += fmt.Sprintf(` \
--search-engine=opensearch \
--opensearch-host=127.0.0.1 \
--opensearch-port=%d \
--opensearch-index-prefix=%s \
--opensearch-timeout=15`, searchPort, projectName)
} else if searchEngine == "elasticsearch" {
searchPort := docker.GetElasticsearchPort(searchVersion)
searchPort := docker.GetElasticsearchPort()
installCmd += fmt.Sprintf(` \
--search-engine=elasticsearch7 \
--elasticsearch-host=127.0.0.1 \
Expand Down Expand Up @@ -1125,7 +1125,7 @@ commands:
fmt.Println()
cli.PrintInfo("Waiting for OpenSearch to be ready...")
dbPort := "33080" // MySQL 8.0 default port
opensearchPort := docker.GetOpenSearchPort(searchVersion)
opensearchPort := docker.GetOpenSearchPort()
opensearchURL := fmt.Sprintf("http://127.0.0.1:%d", opensearchPort)
for i := 0; i < OpenSearchReadinessMaxRetries; i++ {
checkCmd := exec.Command("curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", opensearchURL)
Expand Down
233 changes: 102 additions & 131 deletions internal/docker/compose.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,13 @@ const (
// StandardDBPort is the standard MySQL/MariaDB port (3306) that is additionally
// exposed for the default database service, alongside its version-specific port.
StandardDBPort = 3306
// StandardSearchPort is the standard OpenSearch/Elasticsearch port (9200) that is
// additionally exposed for the default search service, alongside its version-specific port.
// StandardSearchPort is the fixed host port for the single shared OpenSearch
// container (also the container-internal port for both search engines).
StandardSearchPort = 9200
// StandardElasticsearchPort is the fixed host port for the single shared
// Elasticsearch container. It differs from StandardSearchPort so that an
// OpenSearch and an Elasticsearch container can run simultaneously.
StandardElasticsearchPort = 9500
)

// Default RabbitMQ credentials
Expand Down Expand Up @@ -219,7 +223,6 @@ func (g *ComposeGenerator) GenerateGlobalServices(configs []*config.Config) erro

// Only the global default DB version gets the standard port 3306.
// MySQL default takes precedence over MariaDB default if both are configured.
totalSearchServices := len(requiredServices.opensearch) + len(requiredServices.elasticsearch)

// Add MySQL services
for version, svcCfg := range requiredServices.mysql {
Expand All @@ -244,22 +247,19 @@ func (g *ComposeGenerator) GenerateGlobalServices(configs []*config.Config) erro
compose.Services["redis"] = g.getRedisService()
}

// Add OpenSearch services
for version, svcCfg := range requiredServices.opensearch {
serviceName := fmt.Sprintf("opensearch%s", strings.ReplaceAll(version, ".", ""))
addStdPort := (totalSearchServices == 1) || (version == defaultOpenSearch)
compose.Services[serviceName] = g.getOpenSearchService(svcCfg, addStdPort)
compose.Volumes[fmt.Sprintf("opensearch%s_data", strings.ReplaceAll(version, ".", ""))] = ComposeVolume{}
compose.Volumes[fmt.Sprintf("opensearch%s_plugins", strings.ReplaceAll(version, ".", ""))] = ComposeVolume{}
// Add a single shared OpenSearch container for all projects. When projects
// request different versions, the global default wins, else the highest.
if _, osCfg := selectSearchService(requiredServices.opensearch, defaultOpenSearch); osCfg != nil {
compose.Services["opensearch"] = g.getOpenSearchService(osCfg)
compose.Volumes["opensearch_data"] = ComposeVolume{}
compose.Volumes["opensearch_plugins"] = ComposeVolume{}
}

// Add Elasticsearch services
for version, svcCfg := range requiredServices.elasticsearch {
serviceName := fmt.Sprintf("elasticsearch%s", strings.ReplaceAll(version, ".", ""))
addStdPort := len(requiredServices.opensearch) == 0 && ((totalSearchServices == 1) || (version == defaultElasticsearch))
compose.Services[serviceName] = g.getElasticsearchService(svcCfg, addStdPort)
compose.Volumes[fmt.Sprintf("elasticsearch%s_data", strings.ReplaceAll(version, ".", ""))] = ComposeVolume{}
compose.Volumes[fmt.Sprintf("elasticsearch%s_plugins", strings.ReplaceAll(version, ".", ""))] = ComposeVolume{}
// Add a single shared Elasticsearch container for all projects.
if _, esCfg := selectSearchService(requiredServices.elasticsearch, defaultElasticsearch); esCfg != nil {
compose.Services["elasticsearch"] = g.getElasticsearchService(esCfg)
compose.Volumes["elasticsearch_data"] = ComposeVolume{}
compose.Volumes["elasticsearch_plugins"] = ComposeVolume{}
}

// Add RabbitMQ if needed
Expand Down Expand Up @@ -393,6 +393,60 @@ func (g *ComposeGenerator) collectRequiredServices(configs []*config.Config) req
return rs
}

// selectSearchService picks the single version/config to run for one search engine
// (OpenSearch or Elasticsearch) when different projects request different versions.
// The global default version wins if set; otherwise the highest requested version is
// used. The container is provisioned with the largest memory any project requests, so
// the shared container satisfies the most demanding project. Returns ("", nil) when no
// project needs the engine.
func selectSearchService(requested map[string]*config.ServiceConfig, defaultVersion string) (string, *config.ServiceConfig) {
if len(requested) == 0 {
return "", nil
}

version := defaultVersion
if version == "" {
for v := range requested {
if version == "" || compareVersionStrings(v, version) > 0 {
version = v
}
}
}

// Provision for the most demanding project across all requested configs.
memory := ""
for _, c := range requested {
if c != nil && parseMemoryToBytes(c.Memory) > parseMemoryToBytes(memory) {
memory = c.Memory
}
}

return version, &config.ServiceConfig{Enabled: true, Version: version, Memory: memory}
}

// parseMemoryToBytes converts a memory string like "2g", "1024m" or "512k" to bytes.
// An empty or unparseable string returns 0.
func parseMemoryToBytes(s string) int64 {
s = strings.TrimSpace(strings.ToLower(s))
if s == "" {
return 0
}
mult := int64(1)
switch {
case strings.HasSuffix(s, "g"):
mult, s = 1<<30, strings.TrimSuffix(s, "g")
case strings.HasSuffix(s, "m"):
mult, s = 1<<20, strings.TrimSuffix(s, "m")
case strings.HasSuffix(s, "k"):
mult, s = 1<<10, strings.TrimSuffix(s, "k")
}
n, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
if err != nil {
return 0
}
return int64(n * float64(mult))
}

// getMySQLService returns a MySQL service configuration
func (g *ComposeGenerator) getMySQLService(svcCfg *config.ServiceConfig, addStandardPort bool) ComposeService {
version := svcCfg.Version
Expand Down Expand Up @@ -520,11 +574,10 @@ func (g *ComposeGenerator) getValkeyService() ComposeService {
}
}

// getOpenSearchService returns an OpenSearch service configuration
func (g *ComposeGenerator) getOpenSearchService(svcCfg *config.ServiceConfig, addStandardPort bool) ComposeService {
version := svcCfg.Version
imageVersion := ResolveOpenSearchVersion(version)
port := GetOpenSearchPort(imageVersion)
// getOpenSearchService returns the configuration for the single shared OpenSearch
// container used by all projects. The host port is fixed at StandardSearchPort.
func (g *ComposeGenerator) getOpenSearchService(svcCfg *config.ServiceConfig) ComposeService {
imageVersion := ResolveOpenSearchVersion(svcCfg.Version)

// Default to 1GB if not specified
memory := "1g"
Expand All @@ -534,36 +587,31 @@ func (g *ComposeGenerator) getOpenSearchService(svcCfg *config.ServiceConfig, ad

heapSize := fmt.Sprintf("-Xms%s -Xmx%s", memory, memory)

ports := []string{fmt.Sprintf("%d:9200", port)}
if addStandardPort && port != StandardSearchPort {
ports = append(ports, fmt.Sprintf("%d:9200", StandardSearchPort))
}

return ComposeService{
ContainerName: fmt.Sprintf("magebox-opensearch-%s", version),
ContainerName: "magebox-opensearch",
Image: fmt.Sprintf("opensearchproject/opensearch:%s", imageVersion),
Ports: ports,
Ports: []string{fmt.Sprintf("%d:9200", StandardSearchPort)},
Environment: map[string]string{
"discovery.type": "single-node",
"DISABLE_SECURITY_PLUGIN": "true",
"OPENSEARCH_JAVA_OPTS": heapSize,
"cluster.routing.allocation.disk.threshold_enabled": "false",
},
Volumes: []string{
fmt.Sprintf("opensearch%s_data:/usr/share/opensearch/data", strings.ReplaceAll(version, ".", "")),
fmt.Sprintf("opensearch%s_plugins:/usr/share/opensearch/plugins", strings.ReplaceAll(version, ".", "")),
"opensearch_data:/usr/share/opensearch/data",
"opensearch_plugins:/usr/share/opensearch/plugins",
},
Networks: []string{"magebox"},
Restart: "unless-stopped",
Command: "sh -c \"(bin/opensearch-plugin list | grep -q analysis-icu || bin/opensearch-plugin install --batch analysis-icu) && (bin/opensearch-plugin list | grep -q analysis-phonetic || bin/opensearch-plugin install --batch analysis-phonetic) && /usr/share/opensearch/opensearch-docker-entrypoint.sh\"",
}
}

// getElasticsearchService returns an Elasticsearch service configuration
func (g *ComposeGenerator) getElasticsearchService(svcCfg *config.ServiceConfig, addStandardPort bool) ComposeService {
version := svcCfg.Version
imageVersion := ResolveElasticsearchVersion(version)
port := GetElasticsearchPort(imageVersion)
// getElasticsearchService returns the configuration for the single shared
// Elasticsearch container used by all projects. The host port is fixed at
// StandardElasticsearchPort.
func (g *ComposeGenerator) getElasticsearchService(svcCfg *config.ServiceConfig) ComposeService {
imageVersion := ResolveElasticsearchVersion(svcCfg.Version)

// Default to 1GB if not specified
memory := "1g"
Expand All @@ -573,23 +621,18 @@ func (g *ComposeGenerator) getElasticsearchService(svcCfg *config.ServiceConfig,

heapSize := fmt.Sprintf("-Xms%s -Xmx%s", memory, memory)

ports := []string{fmt.Sprintf("%d:9200", port)}
if addStandardPort && port != StandardSearchPort {
ports = append(ports, fmt.Sprintf("%d:9200", StandardSearchPort))
}

return ComposeService{
ContainerName: fmt.Sprintf("magebox-elasticsearch-%s", version),
ContainerName: "magebox-elasticsearch",
Image: fmt.Sprintf("elasticsearch:%s", imageVersion),
Ports: ports,
Ports: []string{fmt.Sprintf("%d:9200", StandardElasticsearchPort)},
Environment: map[string]string{
"discovery.type": "single-node",
"xpack.security.enabled": "false",
"ES_JAVA_OPTS": heapSize,
},
Volumes: []string{
fmt.Sprintf("elasticsearch%s_data:/usr/share/elasticsearch/data", strings.ReplaceAll(version, ".", "")),
fmt.Sprintf("elasticsearch%s_plugins:/usr/share/elasticsearch/plugins", strings.ReplaceAll(version, ".", "")),
"elasticsearch_data:/usr/share/elasticsearch/data",
"elasticsearch_plugins:/usr/share/elasticsearch/plugins",
},
Networks: []string{"magebox"},
Restart: "unless-stopped",
Expand Down Expand Up @@ -753,87 +796,17 @@ func (g *ComposeGenerator) getMariaDBPort(version string) int {
return 33106 // default
}

// normalizeSearchVersion extracts major.minor from a version string like "2.19.4".
// If the input does not contain at least two dot-separated parts, it is returned unchanged.
func normalizeSearchVersion(version string) string {
parts := strings.SplitN(version, ".", 3)
if len(parts) >= 2 {
return parts[0] + "." + parts[1]
}
return version
}

// resolveSearchPortVersion normalizes a search version for port selection. When the
// user provides a major-only version, it first resolves the latest available image
// tag for that major so the selected port matches the resolved major.minor series.
func resolveSearchPortVersion(version string, resolver func(string) string) string {
if !strings.Contains(version, ".") {
return normalizeSearchVersion(resolver(version))
}
return normalizeSearchVersion(version)
}

// computeSearchPort calculates a port from base + major*20 + minor using a
// caller-supplied major.minor version string. If the input cannot be parsed
// numerically, it falls back to basePort. OpenSearch uses base 9200,
// Elasticsearch uses base 9500 to avoid range overlap.
func computeSearchPort(basePort int, majorMinorVersion string) int {
parts := strings.SplitN(majorMinorVersion, ".", 2)
if len(parts) == 2 {
major, err1 := strconv.Atoi(parts[0])
minor, err2 := strconv.Atoi(parts[1])
if err1 == nil && err2 == nil {
return basePort + major*20 + minor
}
}
return basePort
}

// GetOpenSearchPort returns the host port for an OpenSearch version.
// Port convention: 9200 + major*20 + minor (e.g., OS 2.19 → 9259, OS 3.3 → 9263).
func GetOpenSearchPort(version string) int {
normalized := resolveSearchPortVersion(version, ResolveOpenSearchVersion)
ports := map[string]int{
"1.3": 9223,
"2.5": 9245,
"2.10": 9250,
"2.11": 9251,
"2.12": 9252,
"2.13": 9253,
"2.15": 9255,
"2.17": 9257,
"2.19": 9259,
"3.0": 9260,
"3.3": 9263,
}
if port, ok := ports[normalized]; ok {
return port
}
return computeSearchPort(9200, normalized)
// GetOpenSearchPort returns the fixed host port for the single shared OpenSearch
// container. All projects that use OpenSearch share this container and port.
func GetOpenSearchPort() int {
return StandardSearchPort
}

// GetElasticsearchPort returns the host port for an Elasticsearch version.
// Port convention: 9500 + major*20 + minor (e.g., ES 7.17 → 9657, ES 8.11 → 9671).
func GetElasticsearchPort(version string) int {
normalized := resolveSearchPortVersion(version, ResolveElasticsearchVersion)
ports := map[string]int{
"7.6": 9646,
"7.9": 9649,
"7.10": 9650,
"7.16": 9656,
"7.17": 9657,
"8.0": 9660,
"8.4": 9664,
"8.7": 9667,
"8.11": 9671,
"8.14": 9674,
"8.15": 9675,
"8.17": 9677,
}
if port, ok := ports[normalized]; ok {
return port
}
return computeSearchPort(9500, normalized)
// GetElasticsearchPort returns the fixed host port for the single shared
// Elasticsearch container. All projects that use Elasticsearch share this
// container and port.
func GetElasticsearchPort() int {
return StandardElasticsearchPort
}

// ResolveElasticsearchVersion resolves a major.minor version string to the latest available full
Expand Down Expand Up @@ -1095,17 +1068,15 @@ func (g *ComposeGenerator) GenerateDefaultServices(globalCfg *config.GlobalConfi
compose.Services["redis"] = g.getRedisService()
}

// Add OpenSearch if configured
// Add the shared OpenSearch container if configured
if globalCfg.DefaultServices.OpenSearch != "" {
version := globalCfg.DefaultServices.OpenSearch
serviceName := fmt.Sprintf("opensearch%s", strings.ReplaceAll(version, ".", ""))
svcCfg := &config.ServiceConfig{
Enabled: true,
Version: version,
Version: globalCfg.DefaultServices.OpenSearch,
}
compose.Services[serviceName] = g.getOpenSearchService(svcCfg, true)
compose.Volumes[fmt.Sprintf("opensearch%s_data", strings.ReplaceAll(version, ".", ""))] = ComposeVolume{}
compose.Volumes[fmt.Sprintf("opensearch%s_plugins", strings.ReplaceAll(version, ".", ""))] = ComposeVolume{}
compose.Services["opensearch"] = g.getOpenSearchService(svcCfg)
compose.Volumes["opensearch_data"] = ComposeVolume{}
compose.Volumes["opensearch_plugins"] = ComposeVolume{}
}

// Add Mailpit (useful for all projects)
Expand Down
Loading
Loading