From fea02e9b8c553bd0672883213b4255bed90889cc Mon Sep 17 00:00:00 2001 From: chessracer Date: Mon, 16 Apr 2018 13:05:03 -0700 Subject: [PATCH 1/5] DEVOPS-542 allow customized endpoint label --- README.md | 1 + main.go | 16 +++++++++++++--- metadata/metadata.go | 11 +++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index efa49db..f43be05 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ The following environment variables are used to configure global options. |----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------| | POLL_INTERVAL | Value in milliseconds to check for rancher metadata updates | `1000` | | FORCE_UPDATE_INTERVAL | Value in minutes to force a resource poll. Increasing this value may be required if you run into api limits enforced by your cloud providor | `1` | +| SERVICE_LABEL_ENDPOINT | Which label to search for elb names to update. Useful to customize if a single rancher env spans disparate cloud provider accounts, and multiple aws-elb services are required to access each one | `io.rancher.service.external_lb.endpoint` | Contact ======== diff --git a/main.go b/main.go index 80bc8b8..f72ae2b 100644 --- a/main.go +++ b/main.go @@ -19,8 +19,9 @@ import ( ) const ( - EnvVarPollInterval = "POLL_INTERVAL" - EnvVarForceUpdateInterval = "FORCE_UPDATE_INTERVAL" + EnvVarPollInterval = "POLL_INTERVAL" + EnvVarForceUpdateInterval = "FORCE_UPDATE_INTERVAL" + EnvVarServiceLabelEndpoint = "SERVICE_LABEL_ENDPOINT" ) var ( @@ -34,6 +35,7 @@ var ( c *CattleClient targetPoolSuffix string + serviceLabelEndpoint string metadataLBConfigsCached = make(map[string]model.LBConfig) forceUpdateInterval float64 @@ -50,6 +52,14 @@ func setEnv() { var err error + serviceLabelEndpoint = os.Getenv(EnvVarServiceLabelEndpoint) + if len(serviceLabelEndpoint) == 0 { + logrus.Info(EnvVarServiceLabelEndpoint + " is not set") + serviceLabelEndpoint = "" + } else { + logrus.Info(EnvVarServiceLabelEndpoint + " is set to: " + serviceLabelEndpoint) + } + // initialize polling and forceUpdate intervals i = os.Getenv(EnvVarForceUpdateInterval) if i == "" { @@ -143,7 +153,7 @@ func main() { if update || updateForced { // get records from metadata - metadataLBConfigs, err := m.GetMetadataLBConfigs(targetPoolSuffix) + metadataLBConfigs, err := m.GetMetadataLBConfigs(targetPoolSuffix, serviceLabelEndpoint) if err != nil { logrus.Errorf("Failed to get LB configs from metadata: %v", err) continue diff --git a/metadata/metadata.go b/metadata/metadata.go index cbadf14..868c812 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -10,9 +10,9 @@ import ( ) const ( - metadataURLTemplate = "http://%v/2015-12-19" - serviceLabelEndpoint = "io.rancher.service.external_lb.endpoint" - serviceLabelEndpointLegacy = "io.rancher.service.external_lb_endpoint" + metadataURLTemplate = "http://%v/2015-12-19" + serviceLabelEndpointLegacy = "io.rancher.service.external_lb_endpoint" + DefaultServiceLabelEndpoint = "io.rancher.service.external_lb.endpoint" // DefaultMetadataAddress specifies the default value to use if nothing is specified DefaultMetadataAddress = "169.254.169.250" @@ -70,7 +70,10 @@ func (m *MetadataClient) GetVersion() (string, error) { } // GetMetadataLBConfigs ... -func (m *MetadataClient) GetMetadataLBConfigs(targetPoolSuffix string) (map[string]model.LBConfig, error) { +func (m *MetadataClient) GetMetadataLBConfigs(targetPoolSuffix string, serviceLabelEndpoint string) (map[string]model.LBConfig, error) { + if serviceLabelEndpoint == "" { + serviceLabelEndpoint = DefaultServiceLabelEndpoint + } lbConfigs := make(map[string]model.LBConfig) services, err := m.MetadataClient.GetServices() if err != nil { From 70c7342f29a0db270a8d6ae15b86715bc6ac6874 Mon Sep 17 00:00:00 2001 From: chessracer Date: Wed, 25 Apr 2018 09:28:49 -0700 Subject: [PATCH 2/5] fix bad merge from master - it removed the SERVICE_LABEL_ENDPOINT env var --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 186970c..923bb2d 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ const ( EnvVarPollInterval = "POLL_INTERVAL" EnvVarForceUpdateInterval = "FORCE_UPDATE_INTERVAL" EnvVarLBTargetRancherSuffix = "LB_TARGET_RANCHER_SUFFIX" + EnvVarServiceLabelEndpoint = "SERVICE_LABEL_ENDPOINT" ) var ( From 2af3f21cdf1f7fd8f2495ff12db2a1bf4dc4fc9d Mon Sep 17 00:00:00 2001 From: chessracer Date: Fri, 4 May 2018 10:14:21 -0700 Subject: [PATCH 3/5] DEVOPS-556 try restricting metadata call to self stack (optional), in order to prevent cross-stack registrations / deregistrations --- README.md | 5 +++-- main.go | 47 +++++++++++++++++++++++++++++--------------- metadata/metadata.go | 10 +++++++++- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index f43be05..4f4b420 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ external-lb ========== -Rancher service facilitating integration of rancher with external load balancers. This service updates external LB with services created in Rancher that ask to be load balanced using an external LB. +Rancher service facilitating integration of rancher with external load balancers. This service updates external LB with services created in Rancher that ask to be load balanced using an external LB. Initial version comes with f5 BIG-IP support; but a pluggable provider model makes it easy to implement other providers later. Design ========== -* The external-lb gets deployed as a Rancher service containerized app. +* The external-lb gets deployed as a Rancher service containerized app. * It enables any other service to be registered to external LB if the service has exposed a public port and has the label 'io.rancher.service.external_lb_endpoint' @@ -23,6 +23,7 @@ The following environment variables are used to configure global options. | POLL_INTERVAL | Value in milliseconds to check for rancher metadata updates | `1000` | | FORCE_UPDATE_INTERVAL | Value in minutes to force a resource poll. Increasing this value may be required if you run into api limits enforced by your cloud providor | `1` | | SERVICE_LABEL_ENDPOINT | Which label to search for elb names to update. Useful to customize if a single rancher env spans disparate cloud provider accounts, and multiple aws-elb services are required to access each one | `io.rancher.service.external_lb.endpoint` | +| RESTRICT_SERVICE_TO_SELF_STACK | Limit service registration to those running in the same stack as the external-lb service | false | Contact ======== diff --git a/main.go b/main.go index 923bb2d..ac7cad9 100644 --- a/main.go +++ b/main.go @@ -19,13 +19,14 @@ import ( ) const ( - DefaultPollInterval = "1000" - DefaultForceUpdateInterval = "1" - DefaultLBTargetRancherSuffix = "rancher.internal" - EnvVarPollInterval = "POLL_INTERVAL" - EnvVarForceUpdateInterval = "FORCE_UPDATE_INTERVAL" - EnvVarLBTargetRancherSuffix = "LB_TARGET_RANCHER_SUFFIX" - EnvVarServiceLabelEndpoint = "SERVICE_LABEL_ENDPOINT" + DefaultPollInterval = "1000" + DefaultForceUpdateInterval = "1" + DefaultLBTargetRancherSuffix = "rancher.internal" + EnvVarPollInterval = "POLL_INTERVAL" + EnvVarForceUpdateInterval = "FORCE_UPDATE_INTERVAL" + EnvVarLBTargetRancherSuffix = "LB_TARGET_RANCHER_SUFFIX" + EnvVarServiceLabelEndpoint = "SERVICE_LABEL_ENDPOINT" + EnvVarRestrictServiceToSelfStack = "RESTRICT_SERVICE_TO_SELF_STACK" ) var ( @@ -38,14 +39,15 @@ var ( m *metadata.MetadataClient c *CattleClient - targetPoolSuffix string - serviceLabelEndpoint string - metadataLBConfigsCached = make(map[string]model.LBConfig) - - forceUpdateInterval float64 - pollInterval float64 - p string - i string + targetPoolSuffix string + serviceLabelEndpoint string + restrictServiceToSelfStack bool + metadataLBConfigsCached = make(map[string]model.LBConfig) + forceUpdateInterval float64 + pollInterval float64 + p string + i string + r string ) func setEnv() { @@ -56,6 +58,19 @@ func setEnv() { var err error + // optionally restrict the service to those running in same stack as external-lb + r = os.Getenv(EnvVarRestrictServiceToSelfStack) + if len(r) == 0 { + logrus.Info(EnvVarRestrictServiceToSelfStack + " is not set") + restrictServiceToSelfStack = false + } else { + restrictServiceToSelfStack, err = strconv.ParseBool(r) + if err != nil { + logrus.Fatalf("Failed to initialize restrictServiceToSelfStack: %v", err) + } + logrus.Info(EnvVarRestrictServiceToSelfStack + " is set to: " + strconv.FormatBool(restrictServiceToSelfStack)) + } + serviceLabelEndpoint = os.Getenv(EnvVarServiceLabelEndpoint) if len(serviceLabelEndpoint) == 0 { logrus.Info(EnvVarServiceLabelEndpoint + " is not set") @@ -157,7 +172,7 @@ func main() { if update || updateForced { // get records from metadata - metadataLBConfigs, err := m.GetMetadataLBConfigs(targetPoolSuffix, serviceLabelEndpoint) + metadataLBConfigs, err := m.GetMetadataLBConfigs(targetPoolSuffix, serviceLabelEndpoint, restrictServiceToSelfStack) if err != nil { logrus.Errorf("Failed to get LB configs from metadata: %v", err) continue diff --git a/metadata/metadata.go b/metadata/metadata.go index 868c812..77ff294 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -70,12 +70,13 @@ func (m *MetadataClient) GetVersion() (string, error) { } // GetMetadataLBConfigs ... -func (m *MetadataClient) GetMetadataLBConfigs(targetPoolSuffix string, serviceLabelEndpoint string) (map[string]model.LBConfig, error) { +func (m *MetadataClient) GetMetadataLBConfigs(targetPoolSuffix string, serviceLabelEndpoint string, restrictServiceToSelfStack bool) (map[string]model.LBConfig, error) { if serviceLabelEndpoint == "" { serviceLabelEndpoint = DefaultServiceLabelEndpoint } lbConfigs := make(map[string]model.LBConfig) services, err := m.MetadataClient.GetServices() + selfStack, err := m.MetadataClient.GetSelfStack() if err != nil { logrus.Infof("Error reading services: %v", err) } else { @@ -97,6 +98,13 @@ func (m *MetadataClient) GetMetadataLBConfigs(targetPoolSuffix string, serviceLa continue } + // Skip the service if it does not exist in the desired stack (optional) + if restrictServiceToSelfStack && (service.StackName != selfStack.Name) { + logrus.Errorf("Skipping service %s as it is running in a different stack %s than external-lb: %s", + service.Name, service.StackName, selfStack.Name) + continue + } + // get the service port if len(service.Ports) == 0 { logrus.Warnf("Skipping LB configuration for service %s: "+ From eae635f2d7d7a56cba9bd349725364f48b8cd5fc Mon Sep 17 00:00:00 2001 From: chessracer Date: Sun, 6 May 2018 12:28:31 -0700 Subject: [PATCH 4/5] Aligns provider lookup vpcID with EC2 lookup vpc id. This was causing a bug where we were deregistering instances in other vpcs, that our vpc-specific metadata did not describe --- README.md | 2 +- metadata/metadata.go | 2 ++ providers/elbv1/aws_elbv1.go | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f4b420..cb897cf 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ The following environment variables are used to configure global options. | POLL_INTERVAL | Value in milliseconds to check for rancher metadata updates | `1000` | | FORCE_UPDATE_INTERVAL | Value in minutes to force a resource poll. Increasing this value may be required if you run into api limits enforced by your cloud providor | `1` | | SERVICE_LABEL_ENDPOINT | Which label to search for elb names to update. Useful to customize if a single rancher env spans disparate cloud provider accounts, and multiple aws-elb services are required to access each one | `io.rancher.service.external_lb.endpoint` | -| RESTRICT_SERVICE_TO_SELF_STACK | Limit service registration to those running in the same stack as the external-lb service | false | +| RESTRICT_SERVICE_TO_SELF_STACK | Limit service registration to those running in the same stack as the external-lb service | 'false' | Contact ======== diff --git a/metadata/metadata.go b/metadata/metadata.go index 77ff294..499d399 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -99,6 +99,8 @@ func (m *MetadataClient) GetMetadataLBConfigs(targetPoolSuffix string, serviceLa } // Skip the service if it does not exist in the desired stack (optional) + logrus.Debugf("Service %s is running in stack %s", service.Name, service.StackName) + logrus.Debugf("external-lb is running in stack %s", selfStack.Name) if restrictServiceToSelfStack && (service.StackName != selfStack.Name) { logrus.Errorf("Skipping service %s as it is running in a different stack %s than external-lb: %s", service.Name, service.StackName, selfStack.Name) diff --git a/providers/elbv1/aws_elbv1.go b/providers/elbv1/aws_elbv1.go index d701985..65b6c80 100644 --- a/providers/elbv1/aws_elbv1.go +++ b/providers/elbv1/aws_elbv1.go @@ -120,6 +120,7 @@ func (p *AWSELBv1Provider) GetLBConfigs() ([]model.LBConfig, error) { } for _, lb := range allLb { + if _, ok := lbTags[*lb.LoadBalancerName]; !ok { continue } @@ -128,6 +129,13 @@ func (p *AWSELBv1Provider) GetLBConfigs() ([]model.LBConfig, error) { var targetPoolName, servicePort string var ok bool + + logrus.Debugf("Working on lb %s with VPCId: %s", *lb.LoadBalancerName, *lb.VPCId) + if *lb.VPCId != p.vpcID { + logrus.Debugf("Skipping LB whose vpc %s does not match our vpc %s", *lb.VPCId, p.vpcID) + continue + } + if targetPoolName, ok = tags[TagNameTargetPool]; !ok { logrus.Debugf("Skipping LB without targetPool tag: %s", *lb.LoadBalancerName) continue From df31b7982e9ec30154e9fd56966565d4a3717671 Mon Sep 17 00:00:00 2001 From: chessracer Date: Mon, 7 May 2018 10:56:04 -0700 Subject: [PATCH 5/5] change log level to debug for skipping-due-to-different-stack message --- metadata/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata/metadata.go b/metadata/metadata.go index 499d399..b011996 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -102,7 +102,7 @@ func (m *MetadataClient) GetMetadataLBConfigs(targetPoolSuffix string, serviceLa logrus.Debugf("Service %s is running in stack %s", service.Name, service.StackName) logrus.Debugf("external-lb is running in stack %s", selfStack.Name) if restrictServiceToSelfStack && (service.StackName != selfStack.Name) { - logrus.Errorf("Skipping service %s as it is running in a different stack %s than external-lb: %s", + logrus.Debugf("Skipping service %s as it is running in a different stack %s than external-lb: %s", service.Name, service.StackName, selfStack.Name) continue }