diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index 5b48bc1f..19a4dfc4 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -158,7 +158,7 @@ type GitFileGeneratorItem struct { type SCMProviderGenerator struct { // Which provider to use and config for it. Github *SCMProviderGeneratorGithub `json:"github,omitempty"` - // TODO other providers. + Gitlab *SCMProviderGeneratorGitlab `json:"gitlab,omitempty"` // Filters for which repos should be considered. Filters []SCMProviderGeneratorFilter `json:"filters,omitempty"` // Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers @@ -181,6 +181,20 @@ type SCMProviderGeneratorGithub struct { AllBranches bool `json:"allBranches,omitempty"` } +// SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. +type SCMProviderGeneratorGitlab struct { + // Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. + Group string `json:"group"` + // Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + IncludeSubgroups bool `json:"includeSubgroups,omitempty"` + // The Gitlab API URL to talk to. + API string `json:"api,omitempty"` + // Authentication token reference. + TokenRef *SecretRef `json:"tokenRef,omitempty"` + // Scan all branches instead of just the default branch. + AllBranches bool `json:"allBranches,omitempty"` +} + // SCMProviderGeneratorFilter is a single repository filter. // If multiple filter types are set on a single struct, they will be AND'd together. All filters must // pass for a repo to be included. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6860ead5..b0f7f3d0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -462,6 +462,11 @@ func (in *SCMProviderGenerator) DeepCopyInto(out *SCMProviderGenerator) { *out = new(SCMProviderGeneratorGithub) (*in).DeepCopyInto(*out) } + if in.Gitlab != nil { + in, out := &in.Gitlab, &out.Gitlab + *out = new(SCMProviderGeneratorGitlab) + (*in).DeepCopyInto(*out) + } if in.Filters != nil { in, out := &in.Filters, &out.Filters *out = make([]SCMProviderGeneratorFilter, len(*in)) @@ -542,6 +547,26 @@ func (in *SCMProviderGeneratorGithub) DeepCopy() *SCMProviderGeneratorGithub { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SCMProviderGeneratorGitlab) DeepCopyInto(out *SCMProviderGeneratorGitlab) { + *out = *in + if in.TokenRef != nil { + in, out := &in.TokenRef, &out.TokenRef + *out = new(SecretRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SCMProviderGeneratorGitlab. +func (in *SCMProviderGeneratorGitlab) DeepCopy() *SCMProviderGeneratorGitlab { + if in == nil { + return nil + } + out := new(SCMProviderGeneratorGitlab) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretRef) DeepCopyInto(out *SecretRef) { *out = *in diff --git a/docs/Generators-SCM-Provider.md b/docs/Generators-SCM-Provider.md index e696d7b6..eefc48e1 100644 --- a/docs/Generators-SCM-Provider.md +++ b/docs/Generators-SCM-Provider.md @@ -2,7 +2,6 @@ The SCM Provider generator uses the API of an SCMaaS provider (eg GitHub) to automatically discover repositories within an organization. This fits well with GitOps layout patterns that split microservices across many repositories. -Support is currently limited to GitHub, but PRs are welcome to add more SCM providers. ```yaml apiVersion: argoproj.io/v1alpha1 @@ -57,6 +56,45 @@ For label filtering, the repository topics are used. Available clone protocols are `ssh` and `https`. +## Gitlab + +The Gitlab mode uses the Gitlab API to scan and organization in either gitlab.com or self-hosted gitlab. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapps +spec: + generators: + - scmProvider: + gitlab: + # The base Gitlab group to scan. You can either use the group id or the full namespaced path. + group: "8675309" + # For GitHub Enterprise: + api: https://gitlab.example.com/ + # If true, scan every branch of every repository. If false, scan only the default branch. Defaults to false. + allBranches: true + # If true, recurses through subgroups. If false, it searches only in the base group. Defaults to false. + includeSubgroups: true + # Reference to a Secret containing an access token. (optional) + tokenRef: + secretName: gitlab-token + key: token + template: + # ... +``` + +* `group`: Required name of the base Gitlab group to scan. If you have multiple base groups, use multiple generators. +* `api`: If using GitHub Enterprise, the URL to access it. +* `allBranches`: By default (false) the template will only be evaluated for the default branch of each repo. If this is true, every branch of every repository will be passed to the filters. If using this flag, you likely want to use a `branchMatch` filter. +* `allBranches`: By default (false) the controller will only search for repos directly in the base group. If this is true, it will recurse through all the subgroups searching for repos to scan. +* `tokenRef`: A `Secret` name and key containing the Gitlab access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories. + +For label filtering, the repository tags are used. + +Available clone protocols are `ssh` and `https`. + ## Filters Filters allow selecting which repositories to generate for. Each filter can declare one or more conditions, all of which must pass. If multiple filters are present, any can match for a repository to be included. If no filters are specified, all repositories will be processed. diff --git a/go.mod b/go.mod index 1b5a2cb4..bf949ccf 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.6.1 github.com/valyala/fasttemplate v1.2.1 + github.com/xanzy/go-gitlab v0.50.0 golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 k8s.io/api v0.19.2 k8s.io/apimachinery v0.19.2 diff --git a/go.sum b/go.sum index 86942503..2f4ac88a 100644 --- a/go.sum +++ b/go.sum @@ -377,6 +377,12 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.12.2 h1:D0EVSTwQoQOyfY35QNSuPJA4jpZRtkoGYWQMB7XNg5o= github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= +github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -654,6 +660,8 @@ github.com/vmihailenco/msgpack/v5 v5.0.0-beta.5/go.mod h1:MPECSZPg8yittBek5Gq2Mh github.com/vmihailenco/msgpack/v5 v5.1.0/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/xanzy/go-gitlab v0.50.0 h1:t7IoYTrnLSbdEZN7d8X/5zcr+ZM4TZQ2mXa8MqWlAZQ= +github.com/xanzy/go-gitlab v0.50.0/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -786,10 +794,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201022231255-08b38378de70/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201024042810-be3efd7ff127 h1:pZPp9+iYUqwYKLjht0SDBbRCRK/9gAXDy7pz5fRDpjo= golang.org/x/net v0.0.0-20201024042810-be3efd7ff127/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -964,6 +974,7 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 83cf6917..4ed09000 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -3748,8 +3748,8 @@ spec: all protocols. type: string filters: - description: TODO other providers. Filters for - which repos should be considered. + description: Filters for which repos should be + considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter @@ -3805,6 +3805,41 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines + a connection info specific to Gitlab. + properties: + allBranches: + description: Scan all branches instead of + just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: Gitlab group to scan. Required. You + can use either the project id (recommended) + or the full namespaced path. + type: string + includeSubgroups: + description: Recurse through subgroups (true) + or scan only the base group (false). Defaults + to "false" + type: boolean + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 @@ -4646,8 +4681,7 @@ spec: necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos - should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single @@ -4700,6 +4734,40 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection + info specific to Gitlab. + properties: + allBranches: + description: Scan all branches instead of just the default + branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: Gitlab group to scan. Required. You can + use either the project id (recommended) or the full + namespaced path. + type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan + only the base group (false). Defaults to "false" + type: boolean + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 diff --git a/manifests/install-with-argo-cd.yaml b/manifests/install-with-argo-cd.yaml index 8cadb625..38b34b39 100644 --- a/manifests/install-with-argo-cd.yaml +++ b/manifests/install-with-argo-cd.yaml @@ -4623,7 +4623,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: @@ -4669,6 +4669,35 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. + type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 @@ -5316,7 +5345,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: @@ -5362,6 +5391,35 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. + type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 diff --git a/manifests/install.yaml b/manifests/install.yaml index 569dec72..97876c85 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2862,7 +2862,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: @@ -2908,6 +2908,35 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. + type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 @@ -3555,7 +3584,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: @@ -3601,6 +3630,35 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. + type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 diff --git a/pkg/generators/scm_provider.go b/pkg/generators/scm_provider.go index 0dfab3e9..6982f0ff 100644 --- a/pkg/generators/scm_provider.go +++ b/pkg/generators/scm_provider.go @@ -68,6 +68,15 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha if err != nil { return nil, fmt.Errorf("error initializing Github service: %v", err) } + } else if providerConfig.Gitlab != nil { + token, err := g.getSecretRef(ctx, providerConfig.Gitlab.TokenRef, applicationSetInfo.Namespace) + if err != nil { + return nil, fmt.Errorf("error fetching Gitlab token: %v", err) + } + provider, err = scm_provider.NewGitlabProvider(ctx, providerConfig.Gitlab.Group, token, providerConfig.Gitlab.API, providerConfig.Gitlab.AllBranches, providerConfig.Gitlab.IncludeSubgroups) + if err != nil { + return nil, fmt.Errorf("error initializing Gitlab service: %v", err) + } } else { return nil, fmt.Errorf("no SCM provider implementation configured") } diff --git a/pkg/services/scm_provider/gitlab.go b/pkg/services/scm_provider/gitlab.go new file mode 100644 index 00000000..433abfde --- /dev/null +++ b/pkg/services/scm_provider/gitlab.go @@ -0,0 +1,131 @@ +package scm_provider + +import ( + "context" + "fmt" + "os" + + gitlab "github.com/xanzy/go-gitlab" +) + +type GitlabProvider struct { + client *gitlab.Client + organization string + allBranches bool + includeSubgroups bool +} + +var _ SCMProviderService = &GitlabProvider{} + +func NewGitlabProvider(ctx context.Context, organization string, token string, url string, allBranches, includeSubgroups bool) (*GitlabProvider, error) { + // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. + if token == "" { + token = os.Getenv("GITLAB_TOKEN") + } + var client *gitlab.Client + if url == "" { + var err error + client, err = gitlab.NewClient(token) + if err != nil { + return nil, err + } + } else { + var err error + client, err = gitlab.NewClient(token, gitlab.WithBaseURL(url)) + if err != nil { + return nil, err + } + } + return &GitlabProvider{client: client, organization: organization, allBranches: allBranches, includeSubgroups: includeSubgroups}, nil +} + +func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([]*Repository, error) { + opt := &gitlab.ListGroupProjectsOptions{ + ListOptions: gitlab.ListOptions{PerPage: 100}, + IncludeSubgroups: &g.includeSubgroups, + } + repos := []*Repository{} + for { + gitlabRepos, resp, err := g.client.Groups.ListGroupProjects(g.organization, opt) + if err != nil { + return nil, fmt.Errorf("error listing projects for %s: %v", g.organization, err) + } + for _, gitlabRepo := range gitlabRepos { + var url string + switch cloneProtocol { + // Default to SSH if unspecified (i.e. if ""). + case "", "ssh": + url = gitlabRepo.SSHURLToRepo + case "https": + url = gitlabRepo.HTTPURLToRepo + default: + return nil, fmt.Errorf("unknown clone protocol for Gitlab %v", cloneProtocol) + } + + branches, err := g.listBranches(ctx, gitlabRepo) + if err != nil { + return nil, fmt.Errorf("error listing branches for %s/%s: %v", g.organization, gitlabRepo.Name, err) + } + + for _, branch := range branches { + repos = append(repos, &Repository{ + Organization: gitlabRepo.Namespace.FullPath, + Repository: gitlabRepo.Path, + URL: url, + Branch: branch, + Labels: gitlabRepo.TagList, + }) + } + } + if resp.CurrentPage >= resp.TotalPages { + break + } + opt.Page = resp.NextPage + } + return repos, nil +} + +func (g *GitlabProvider) RepoHasPath(_ context.Context, repo *Repository, path string) (bool, error) { + p, _, err := g.client.Projects.GetProject(repo.Organization+"/"+repo.Repository, nil) + if err != nil { + return false, err + } + _, resp, err := g.client.Repositories.ListTree(p.ID, &gitlab.ListTreeOptions{ + Path: &path, + Ref: &repo.Branch, + }) + if resp.TotalItems == 0 { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (g *GitlabProvider) listBranches(_ context.Context, repo *gitlab.Project) ([]string, error) { + // If we don't specifically want to query for all branches, just use the default branch and call it a day. + if !g.allBranches { + return []string{repo.DefaultBranch}, nil + } + // Otherwise, scrape the ListBranches API. + opt := &gitlab.ListBranchesOptions{ + ListOptions: gitlab.ListOptions{PerPage: 100}, + } + branches := []string{} + for { + gitlabBranches, resp, err := g.client.Branches.ListBranches(repo.ID, opt) + if err != nil { + return nil, err + } + for _, gitlabBranch := range gitlabBranches { + branches = append(branches, gitlabBranch.Name) + } + + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return branches, nil +} diff --git a/pkg/services/scm_provider/gitlab_test.go b/pkg/services/scm_provider/gitlab_test.go new file mode 100644 index 00000000..abb447f8 --- /dev/null +++ b/pkg/services/scm_provider/gitlab_test.go @@ -0,0 +1,86 @@ +package scm_provider + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGitlabListRepos(t *testing.T) { + cases := []struct { + name, proto, url string + hasError, allBranches, includeSubgroups bool + branches []string + }{ + { + name: "blank protocol", + url: "git@gitlab.com:test-argocd-proton/argocd.git", + branches: []string{"master"}, + }, + { + name: "ssh protocol", + proto: "ssh", + url: "git@gitlab.com:test-argocd-proton/argocd.git", + }, + { + name: "https protocol", + proto: "https", + url: "https://gitlab.com/test-argocd-proton/argocd.git", + }, + { + name: "other protocol", + proto: "other", + hasError: true, + }, + { + name: "all branches", + allBranches: true, + url: "git@gitlab.com:test-argocd-proton/argocd.git", + branches: []string{"master", "pipeline-1310077506"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + provider, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", c.allBranches, c.includeSubgroups) + rawRepos, err := provider.ListRepos(context.Background(), c.proto) + if c.hasError { + assert.NotNil(t, err) + } else { + checkRateLimit(t, err) + assert.Nil(t, err) + // Just check that this one project shows up. Not a great test but better thing nothing? + repos := []*Repository{} + branches := []string{} + for _, r := range rawRepos { + if r.Repository == "argocd" { + repos = append(repos, r) + branches = append(branches, r.Branch) + } + } + assert.NotEmpty(t, repos) + assert.Equal(t, c.url, repos[0].URL) + for _, b := range c.branches { + assert.Contains(t, branches, b) + } + } + }) + } +} + +func TestGitlabHasPath(t *testing.T) { + host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", false, true) + repo := &Repository{ + Organization: "test-argocd-proton", + Repository: "argocd", + Branch: "master", + } + ok, err := host.RepoHasPath(context.Background(), repo, "argocd") + assert.Nil(t, err) + assert.True(t, ok) + + ok, err = host.RepoHasPath(context.Background(), repo, "notathing") + assert.Nil(t, err) + assert.False(t, ok) +}