From ad39f4bcec6a7f4e13ecd5079c705d936e150473 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Wed, 30 Jun 2021 13:35:49 -0400 Subject: [PATCH 01/14] added gitlab support --- api/v1alpha1/applicationset_types.go | 13 +++ go.mod | 1 + go.sum | 10 ++ pkg/services/scm_provider/gitlab.go | 135 +++++++++++++++++++++++ pkg/services/scm_provider/gitlab_test.go | 99 +++++++++++++++++ 5 files changed, 258 insertions(+) create mode 100644 pkg/services/scm_provider/gitlab.go create mode 100644 pkg/services/scm_provider/gitlab_test.go diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index 5b48bc1f..5facd971 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -158,6 +158,7 @@ type GitFileGeneratorItem struct { type SCMProviderGenerator struct { // Which provider to use and config for it. Github *SCMProviderGeneratorGithub `json:"github,omitempty"` + Gitlab *SCMProviderGeneratorGitlab `json:"gitlab,omitempty"` // TODO other providers. // Filters for which repos should be considered. Filters []SCMProviderGeneratorFilter `json:"filters,omitempty"` @@ -181,6 +182,18 @@ type SCMProviderGeneratorGithub struct { AllBranches bool `json:"allBranches,omitempty"` } +// SCMProviderGeneratorGitlab defines a connection info specific to GitHub. +type SCMProviderGeneratorGitlab struct { + // GitHub org to scan. Required. + Group string `json:"group"` + // The GitHub API URL to talk to. If blank, use https://api.github.com/. + API string `json:"api,omitempty"` + // Authentication token reference. + TokenRef *SecretRef `json:"tokenRef,omitempty"` + // Scan all branches instead of just the default branch. + AllBranches bool `json:"allBranches,omitempty"` +} + // SCMProviderGeneratorFilter is a single repository filter. // If multiple filter types are set on a single struct, they will be AND'd together. All filters must // pass for a repo to be included. diff --git a/go.mod b/go.mod index 1b5a2cb4..1b6f75dc 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.6.1 github.com/valyala/fasttemplate v1.2.1 + github.com/xanzy/go-gitlab v0.50.0 // indirect golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 k8s.io/api v0.19.2 k8s.io/apimachinery v0.19.2 diff --git a/go.sum b/go.sum index 86942503..c8930e0e 100644 --- a/go.sum +++ b/go.sum @@ -377,6 +377,11 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.12.2 h1:D0EVSTwQoQOyfY35QNSuPJA4jpZRtkoGYWQMB7XNg5o= github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= +github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -654,6 +659,8 @@ github.com/vmihailenco/msgpack/v5 v5.0.0-beta.5/go.mod h1:MPECSZPg8yittBek5Gq2Mh github.com/vmihailenco/msgpack/v5 v5.1.0/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/xanzy/go-gitlab v0.50.0 h1:t7IoYTrnLSbdEZN7d8X/5zcr+ZM4TZQ2mXa8MqWlAZQ= +github.com/xanzy/go-gitlab v0.50.0/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -786,10 +793,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201022231255-08b38378de70/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201024042810-be3efd7ff127 h1:pZPp9+iYUqwYKLjht0SDBbRCRK/9gAXDy7pz5fRDpjo= golang.org/x/net v0.0.0-20201024042810-be3efd7ff127/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -964,6 +973,7 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= diff --git a/pkg/services/scm_provider/gitlab.go b/pkg/services/scm_provider/gitlab.go new file mode 100644 index 00000000..36d698d4 --- /dev/null +++ b/pkg/services/scm_provider/gitlab.go @@ -0,0 +1,135 @@ +package scm_provider + +import ( + "context" + "fmt" + "os" + + gitlab "github.com/xanzy/go-gitlab" +) + +type GitlabProvider struct { + client *gitlab.Client + organization string + allBranches bool +} + +var _ SCMProviderService = &GitlabProvider{} + +func newTrue() *bool { + b := true + return &b +} + +func NewGitlabProvider(ctx context.Context, organization string, token string, url string, allBranches bool) (*GitlabProvider, error) { + // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. + if token == "" { + token = os.Getenv("GITLAB_TOKEN") + } + var client *gitlab.Client + if url == "" { + var err error + client, err = gitlab.NewClient(token) + if err != nil { + return nil, err + } + } else { + var err error + client, err = gitlab.NewClient(token, gitlab.WithBaseURL(url)) + if err != nil { + return nil, err + } + } + return &GitlabProvider{client: client, organization: organization, allBranches: allBranches}, nil +} + +func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([]*Repository, error) { + opt := &gitlab.ListGroupProjectsOptions{ + ListOptions: gitlab.ListOptions{PerPage: 100}, + IncludeSubgroups: newTrue(), + } + repos := []*Repository{} + for { + gitlabRepos, resp, err := g.client.Groups.ListGroupProjects(g.organization, opt) + if err != nil { + return nil, fmt.Errorf("error listing projects for %s: %v", g.organization, err) + } + for _, gitlabRepo := range gitlabRepos { + var url string + switch cloneProtocol { + // Default to SSH if unspecified (i.e. if ""). + case "", "ssh": + url = gitlabRepo.SSHURLToRepo + case "https": + url = gitlabRepo.HTTPURLToRepo + default: + return nil, fmt.Errorf("unknown clone protocol for Gitlab %v", cloneProtocol) + } + + branches, err := g.listBranches(ctx, gitlabRepo) + if err != nil { + return nil, fmt.Errorf("error listing branches for %s/%s: %v", g.organization, gitlabRepo.Name, err) + } + + for _, branch := range branches { + repos = append(repos, &Repository{ + Organization: g.organization, + Repository: gitlabRepo.PathWithNamespace, + URL: url, + Branch: branch, + Labels: gitlabRepo.TagList, + }) + } + } + if resp.CurrentPage >= resp.TotalPages { + break + } + opt.Page = resp.NextPage + } + return repos, nil +} + +func (g *GitlabProvider) RepoHasPath(ctx context.Context, repo *Repository, path string) (bool, error) { + p, _, err := g.client.Projects.GetProject(repo.Repository, nil) + if err != nil { + return false, fmt.Errorf("Error retrieving project %s", repo.Repository) + } + _, resp, err := g.client.RepositoryFiles.GetFileMetaData(p.ID, path, &gitlab.GetFileMetaDataOptions{ + Ref: &repo.Branch, + }) + // 404s are not an error here, just a normal false. + if resp != nil && resp.StatusCode == 404 { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (g *GitlabProvider) listBranches(ctx context.Context, repo *gitlab.Project) ([]string, error) { + // If we don't specifically want to query for all branches, just use the default branch and call it a day. + if !g.allBranches { + return []string{repo.DefaultBranch}, nil + } + // Otherwise, scrape the ListBranches API. + opt := &gitlab.ListBranchesOptions{ + ListOptions: gitlab.ListOptions{PerPage: 100}, + } + branches := []string{} + for { + gitlabBranches, resp, err := g.client.Branches.ListBranches(repo.ID, opt) + if err != nil { + return nil, err + } + for _, gitlabBranch := range gitlabBranches { + branches = append(branches, gitlabBranch.Name) + } + + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return branches, nil +} diff --git a/pkg/services/scm_provider/gitlab_test.go b/pkg/services/scm_provider/gitlab_test.go new file mode 100644 index 00000000..852ca7da --- /dev/null +++ b/pkg/services/scm_provider/gitlab_test.go @@ -0,0 +1,99 @@ +package scm_provider + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +// func checkRateLimit(t *testing.T, err error) { +// // Check if we've hit a rate limit, don't fail the test if so. +// if err != nil && strings.Contains(err.Error(), "rate limit exceeded") { +// allowRateLimitErrors := os.Getenv("CI") == "" +// t.Logf("Got a rate limit error, consider setting $Gitlab_TOKEN to increase your Gitlab API rate limit: %v\n", err) +// if allowRateLimitErrors { +// t.SkipNow() +// } else { +// t.FailNow() +// } +// } +// } + +func TestGitlabListRepos(t *testing.T) { + cases := []struct { + name, proto, url string + hasError, allBranches bool + branches []string + }{ + { + name: "blank protocol", + url: "git@gitlab.com:test-argocd-proton/argocd.git", + branches: []string{"master"}, + }, + { + name: "ssh protocol", + proto: "ssh", + url: "git@gitlab.com:test-argocd-proton/argocd.git", + }, + { + name: "https protocol", + proto: "https", + url: "https://gitlab.com/test-argocd-proton/argocd.git", + }, + { + name: "other protocol", + proto: "other", + hasError: true, + }, + { + name: "all branches", + allBranches: true, + url: "git@gitlab.com:test-argocd-proton/argocd.git", + branches: []string{"master", "pipeline-1310077506"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + provider, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", c.allBranches) + rawRepos, err := provider.ListRepos(context.Background(), c.proto) + if c.hasError { + assert.NotNil(t, err) + } else { + checkRateLimit(t, err) + assert.Nil(t, err) + // Just check that this one project shows up. Not a great test but better thing nothing? + repos := []*Repository{} + branches := []string{} + for _, r := range rawRepos { + if r.Repository == "test-argocd-proton/argocd" { + repos = append(repos, r) + branches = append(branches, r.Branch) + } + } + assert.NotEmpty(t, repos) + assert.Equal(t, c.url, repos[0].URL) + for _, b := range c.branches { + assert.Contains(t, branches, b) + } + } + }) + } +} + +func TestGitlabHasPath(t *testing.T) { + host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", false) + repo := &Repository{ + Organization: "test-argocd-proton", + Repository: "test-argocd-proton/argocd", + Branch: "master", + } + ok, err := host.RepoHasPath(context.Background(), repo, "templates/") + assert.Nil(t, err) + assert.True(t, ok) + + ok, err = host.RepoHasPath(context.Background(), repo, "notathing/") + assert.Nil(t, err) + assert.False(t, ok) +} From dbe2c618be65adcad82a77cafdf83ebac2b9425f Mon Sep 17 00:00:00 2001 From: empath-nirvana <86739375+empath-nirvana@users.noreply.github.com> Date: Wed, 30 Jun 2021 15:32:00 -0400 Subject: [PATCH 02/14] Update pkg/services/scm_provider/gitlab.go Co-authored-by: Noah Kantrowitz --- pkg/services/scm_provider/gitlab.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/scm_provider/gitlab.go b/pkg/services/scm_provider/gitlab.go index 36d698d4..23c87658 100644 --- a/pkg/services/scm_provider/gitlab.go +++ b/pkg/services/scm_provider/gitlab.go @@ -10,7 +10,7 @@ import ( type GitlabProvider struct { client *gitlab.Client - organization string + group string allBranches bool } From 3b81c83d7a2a19f47af9fd40878add81ae20ba40 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Thu, 1 Jul 2021 12:20:28 -0400 Subject: [PATCH 03/14] fixed path search --- api/v1alpha1/applicationset_types.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 25 +++++++++ .../crds/argoproj.io_applicationsets.yaml | 56 +++++++++++++++++++ manifests/install-with-argo-cd.yaml | 52 +++++++++++++++++ manifests/install.yaml | 52 +++++++++++++++++ pkg/generators/scm_provider.go | 9 +++ pkg/services/scm_provider/gitlab.go | 18 +++--- pkg/services/scm_provider/gitlab_test.go | 2 +- 8 files changed, 206 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index 5facd971..08047b86 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -186,7 +186,7 @@ type SCMProviderGeneratorGithub struct { type SCMProviderGeneratorGitlab struct { // GitHub org to scan. Required. Group string `json:"group"` - // The GitHub API URL to talk to. If blank, use https://api.github.com/. + // The Gitlab API URL to talk to. API string `json:"api,omitempty"` // Authentication token reference. TokenRef *SecretRef `json:"tokenRef,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6860ead5..b0f7f3d0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -462,6 +462,11 @@ func (in *SCMProviderGenerator) DeepCopyInto(out *SCMProviderGenerator) { *out = new(SCMProviderGeneratorGithub) (*in).DeepCopyInto(*out) } + if in.Gitlab != nil { + in, out := &in.Gitlab, &out.Gitlab + *out = new(SCMProviderGeneratorGitlab) + (*in).DeepCopyInto(*out) + } if in.Filters != nil { in, out := &in.Filters, &out.Filters *out = make([]SCMProviderGeneratorFilter, len(*in)) @@ -542,6 +547,26 @@ func (in *SCMProviderGeneratorGithub) DeepCopy() *SCMProviderGeneratorGithub { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SCMProviderGeneratorGitlab) DeepCopyInto(out *SCMProviderGeneratorGitlab) { + *out = *in + if in.TokenRef != nil { + in, out := &in.TokenRef, &out.TokenRef + *out = new(SecretRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SCMProviderGeneratorGitlab. +func (in *SCMProviderGeneratorGitlab) DeepCopy() *SCMProviderGeneratorGitlab { + if in == nil { + return nil + } + out := new(SCMProviderGeneratorGitlab) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretRef) DeepCopyInto(out *SecretRef) { *out = *in diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 83cf6917..50ebfa9a 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -3805,6 +3805,34 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines + a connection info specific to GitHub. + properties: + allBranches: + description: Scan all branches instead of + just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: GitHub org to scan. Required. + type: string + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 @@ -4700,6 +4728,34 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection + info specific to GitHub. + properties: + allBranches: + description: Scan all branches instead of just the default + branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: GitHub org to scan. Required. + type: string + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 diff --git a/manifests/install-with-argo-cd.yaml b/manifests/install-with-argo-cd.yaml index 8cadb625..7d725518 100644 --- a/manifests/install-with-argo-cd.yaml +++ b/manifests/install-with-argo-cd.yaml @@ -4669,6 +4669,32 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: GitHub org to scan. Required. + type: string + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 @@ -5362,6 +5388,32 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: GitHub org to scan. Required. + type: string + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 diff --git a/manifests/install.yaml b/manifests/install.yaml index 569dec72..2cb73d17 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2908,6 +2908,32 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: GitHub org to scan. Required. + type: string + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 @@ -3601,6 +3627,32 @@ spec: required: - organization type: object + gitlab: + description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + properties: + allBranches: + description: Scan all branches instead of just the default branch. + type: boolean + api: + description: The Gitlab API URL to talk to. + type: string + group: + description: GitHub org to scan. Required. + type: string + tokenRef: + description: Authentication token reference. + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - group + type: object requeueAfterSeconds: description: Standard parameters. format: int64 diff --git a/pkg/generators/scm_provider.go b/pkg/generators/scm_provider.go index 0dfab3e9..53043416 100644 --- a/pkg/generators/scm_provider.go +++ b/pkg/generators/scm_provider.go @@ -68,6 +68,15 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha if err != nil { return nil, fmt.Errorf("error initializing Github service: %v", err) } + } else if providerConfig.Gitlab != nil { + token, err := g.getSecretRef(ctx, providerConfig.Gitlab.TokenRef, applicationSetInfo.Namespace) + if err != nil { + return nil, fmt.Errorf("error fetching Gitlab token: %v", err) + } + provider, err = scm_provider.NewGitlabProvider(ctx, providerConfig.Gitlab.Group, token, providerConfig.Gitlab.API, providerConfig.Gitlab.AllBranches) + if err != nil { + return nil, fmt.Errorf("error initializing Gitlab service: %v", err) + } } else { return nil, fmt.Errorf("no SCM provider implementation configured") } diff --git a/pkg/services/scm_provider/gitlab.go b/pkg/services/scm_provider/gitlab.go index 36d698d4..16088d63 100644 --- a/pkg/services/scm_provider/gitlab.go +++ b/pkg/services/scm_provider/gitlab.go @@ -73,8 +73,8 @@ func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([ for _, branch := range branches { repos = append(repos, &Repository{ - Organization: g.organization, - Repository: gitlabRepo.PathWithNamespace, + Organization: gitlabRepo.Namespace.FullPath, + Repository: gitlabRepo.Path, URL: url, Branch: branch, Labels: gitlabRepo.TagList, @@ -90,15 +90,17 @@ func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([ } func (g *GitlabProvider) RepoHasPath(ctx context.Context, repo *Repository, path string) (bool, error) { - p, _, err := g.client.Projects.GetProject(repo.Repository, nil) + p, _, err := g.client.Projects.GetProject(repo.Organization+"/"+repo.Repository, nil) if err != nil { - return false, fmt.Errorf("Error retrieving project %s", repo.Repository) + return false, err } - _, resp, err := g.client.RepositoryFiles.GetFileMetaData(p.ID, path, &gitlab.GetFileMetaDataOptions{ - Ref: &repo.Branch, + _, resp, err := g.client.Repositories.ListTree(p.ID, &gitlab.ListTreeOptions{ + Path: &path, + Ref: &repo.Branch, }) - // 404s are not an error here, just a normal false. - if resp != nil && resp.StatusCode == 404 { + fmt.Printf("%+v\n", err) + fmt.Printf("%+v\n", resp) + if resp.TotalItems == 0 { return false, nil } if err != nil { diff --git a/pkg/services/scm_provider/gitlab_test.go b/pkg/services/scm_provider/gitlab_test.go index 852ca7da..c61a1b71 100644 --- a/pkg/services/scm_provider/gitlab_test.go +++ b/pkg/services/scm_provider/gitlab_test.go @@ -89,7 +89,7 @@ func TestGitlabHasPath(t *testing.T) { Repository: "test-argocd-proton/argocd", Branch: "master", } - ok, err := host.RepoHasPath(context.Background(), repo, "templates/") + ok, err := host.RepoHasPath(context.Background(), repo, "argocd/") assert.Nil(t, err) assert.True(t, ok) From 331f38d74eaa3109b195d84df88778521eda6c57 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Thu, 1 Jul 2021 12:21:17 -0400 Subject: [PATCH 04/14] fixed merge --- pkg/services/scm_provider/gitlab.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/scm_provider/gitlab.go b/pkg/services/scm_provider/gitlab.go index b2a32c49..16088d63 100644 --- a/pkg/services/scm_provider/gitlab.go +++ b/pkg/services/scm_provider/gitlab.go @@ -10,7 +10,7 @@ import ( type GitlabProvider struct { client *gitlab.Client - group string + organization string allBranches bool } From 1b4bf586ba73d6e05db7cb7b993a29575e124392 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Fri, 2 Jul 2021 09:38:31 -0400 Subject: [PATCH 05/14] fixed tests --- pkg/services/scm_provider/gitlab_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/services/scm_provider/gitlab_test.go b/pkg/services/scm_provider/gitlab_test.go index c61a1b71..3bfc2633 100644 --- a/pkg/services/scm_provider/gitlab_test.go +++ b/pkg/services/scm_provider/gitlab_test.go @@ -67,7 +67,7 @@ func TestGitlabListRepos(t *testing.T) { repos := []*Repository{} branches := []string{} for _, r := range rawRepos { - if r.Repository == "test-argocd-proton/argocd" { + if r.Repository == "argocd" { repos = append(repos, r) branches = append(branches, r.Branch) } @@ -86,14 +86,14 @@ func TestGitlabHasPath(t *testing.T) { host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", false) repo := &Repository{ Organization: "test-argocd-proton", - Repository: "test-argocd-proton/argocd", + Repository: "argocd", Branch: "master", } - ok, err := host.RepoHasPath(context.Background(), repo, "argocd/") + ok, err := host.RepoHasPath(context.Background(), repo, "argocd") assert.Nil(t, err) assert.True(t, ok) - ok, err = host.RepoHasPath(context.Background(), repo, "notathing/") + ok, err = host.RepoHasPath(context.Background(), repo, "notathing") assert.Nil(t, err) assert.False(t, ok) } From 08ec55af6a14db1ec67e0ede32226a6b20f69fab Mon Sep 17 00:00:00 2001 From: John Thompson Date: Fri, 2 Jul 2021 09:40:59 -0400 Subject: [PATCH 06/14] removed rate limit check --- pkg/services/scm_provider/gitlab_test.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pkg/services/scm_provider/gitlab_test.go b/pkg/services/scm_provider/gitlab_test.go index 3bfc2633..1c38dd5b 100644 --- a/pkg/services/scm_provider/gitlab_test.go +++ b/pkg/services/scm_provider/gitlab_test.go @@ -7,19 +7,6 @@ import ( "github.com/stretchr/testify/assert" ) -// func checkRateLimit(t *testing.T, err error) { -// // Check if we've hit a rate limit, don't fail the test if so. -// if err != nil && strings.Contains(err.Error(), "rate limit exceeded") { -// allowRateLimitErrors := os.Getenv("CI") == "" -// t.Logf("Got a rate limit error, consider setting $Gitlab_TOKEN to increase your Gitlab API rate limit: %v\n", err) -// if allowRateLimitErrors { -// t.SkipNow() -// } else { -// t.FailNow() -// } -// } -// } - func TestGitlabListRepos(t *testing.T) { cases := []struct { name, proto, url string From 6a35972f096d9c2b6dc3c8e3391257e78eaebe77 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Fri, 2 Jul 2021 11:56:42 -0400 Subject: [PATCH 07/14] tidy --- go.mod | 2 +- go.sum | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1b6f75dc..bf949ccf 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.6.1 github.com/valyala/fasttemplate v1.2.1 - github.com/xanzy/go-gitlab v0.50.0 // indirect + github.com/xanzy/go-gitlab v0.50.0 golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 k8s.io/api v0.19.2 k8s.io/apimachinery v0.19.2 diff --git a/go.sum b/go.sum index c8930e0e..2f4ac88a 100644 --- a/go.sum +++ b/go.sum @@ -379,6 +379,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.12.2 h1:D0EVSTwQoQOyfY35QNSuPJA4jpZRtk github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= From 1c898b13031c044342c2fb812c6c84bb84986c1f Mon Sep 17 00:00:00 2001 From: John Thompson Date: Tue, 6 Jul 2021 09:52:22 -0400 Subject: [PATCH 08/14] fixed github references in comments --- api/v1alpha1/applicationset_types.go | 4 ++-- manifests/crds/argoproj.io_applicationsets.yaml | 8 ++++---- manifests/install-with-argo-cd.yaml | 8 ++++---- manifests/install.yaml | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index 08047b86..e37e744b 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -182,9 +182,9 @@ type SCMProviderGeneratorGithub struct { AllBranches bool `json:"allBranches,omitempty"` } -// SCMProviderGeneratorGitlab defines a connection info specific to GitHub. +// SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. type SCMProviderGeneratorGitlab struct { - // GitHub org to scan. Required. + // Gitlab org to scan. Required. Group string `json:"group"` // The Gitlab API URL to talk to. API string `json:"api,omitempty"` diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 50ebfa9a..44f1ccf6 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -3807,7 +3807,7 @@ spec: type: object gitlab: description: SCMProviderGeneratorGitlab defines - a connection info specific to GitHub. + a connection info specific to Gitlab. properties: allBranches: description: Scan all branches instead of @@ -3817,7 +3817,7 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: GitHub org to scan. Required. + description: Gitlab org to scan. Required. type: string tokenRef: description: Authentication token reference. @@ -4730,7 +4730,7 @@ spec: type: object gitlab: description: SCMProviderGeneratorGitlab defines a connection - info specific to GitHub. + info specific to Gitlab. properties: allBranches: description: Scan all branches instead of just the default @@ -4740,7 +4740,7 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: GitHub org to scan. Required. + description: Gitlab org to scan. Required. type: string tokenRef: description: Authentication token reference. diff --git a/manifests/install-with-argo-cd.yaml b/manifests/install-with-argo-cd.yaml index 7d725518..a810ad82 100644 --- a/manifests/install-with-argo-cd.yaml +++ b/manifests/install-with-argo-cd.yaml @@ -4670,7 +4670,7 @@ spec: - organization type: object gitlab: - description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. properties: allBranches: description: Scan all branches instead of just the default branch. @@ -4679,7 +4679,7 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: GitHub org to scan. Required. + description: Gitlab org to scan. Required. type: string tokenRef: description: Authentication token reference. @@ -5389,7 +5389,7 @@ spec: - organization type: object gitlab: - description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. properties: allBranches: description: Scan all branches instead of just the default branch. @@ -5398,7 +5398,7 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: GitHub org to scan. Required. + description: Gitlab org to scan. Required. type: string tokenRef: description: Authentication token reference. diff --git a/manifests/install.yaml b/manifests/install.yaml index 2cb73d17..6bd801e9 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2909,7 +2909,7 @@ spec: - organization type: object gitlab: - description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. properties: allBranches: description: Scan all branches instead of just the default branch. @@ -2918,7 +2918,7 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: GitHub org to scan. Required. + description: Gitlab org to scan. Required. type: string tokenRef: description: Authentication token reference. @@ -3628,7 +3628,7 @@ spec: - organization type: object gitlab: - description: SCMProviderGeneratorGitlab defines a connection info specific to GitHub. + description: SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. properties: allBranches: description: Scan all branches instead of just the default branch. @@ -3637,7 +3637,7 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: GitHub org to scan. Required. + description: Gitlab org to scan. Required. type: string tokenRef: description: Authentication token reference. From b45c317fb2ba91a01c4c76bc797fdfb0f7e006af Mon Sep 17 00:00:00 2001 From: John Thompson Date: Wed, 7 Jul 2021 10:15:42 -0400 Subject: [PATCH 09/14] added includeSubgroup option, fixed lints --- api/v1alpha1/applicationset_types.go | 4 +++- pkg/generators/scm_provider.go | 2 +- pkg/services/scm_provider/gitlab.go | 22 +++++++++------------- pkg/services/scm_provider/gitlab_test.go | 10 +++++----- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index e37e744b..7545aa3c 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -184,8 +184,10 @@ type SCMProviderGeneratorGithub struct { // SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. type SCMProviderGeneratorGitlab struct { - // Gitlab org to scan. Required. + // Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. Group string `json:"group"` + // Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + IncludeSubgroups bool `json:"includeSubgroups,omitempty"` // The Gitlab API URL to talk to. API string `json:"api,omitempty"` // Authentication token reference. diff --git a/pkg/generators/scm_provider.go b/pkg/generators/scm_provider.go index 53043416..6982f0ff 100644 --- a/pkg/generators/scm_provider.go +++ b/pkg/generators/scm_provider.go @@ -73,7 +73,7 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha if err != nil { return nil, fmt.Errorf("error fetching Gitlab token: %v", err) } - provider, err = scm_provider.NewGitlabProvider(ctx, providerConfig.Gitlab.Group, token, providerConfig.Gitlab.API, providerConfig.Gitlab.AllBranches) + provider, err = scm_provider.NewGitlabProvider(ctx, providerConfig.Gitlab.Group, token, providerConfig.Gitlab.API, providerConfig.Gitlab.AllBranches, providerConfig.Gitlab.IncludeSubgroups) if err != nil { return nil, fmt.Errorf("error initializing Gitlab service: %v", err) } diff --git a/pkg/services/scm_provider/gitlab.go b/pkg/services/scm_provider/gitlab.go index 16088d63..8db9b8d7 100644 --- a/pkg/services/scm_provider/gitlab.go +++ b/pkg/services/scm_provider/gitlab.go @@ -9,19 +9,15 @@ import ( ) type GitlabProvider struct { - client *gitlab.Client - organization string - allBranches bool + client *gitlab.Client + organization string + allBranches bool + includeSubgroups bool } var _ SCMProviderService = &GitlabProvider{} -func newTrue() *bool { - b := true - return &b -} - -func NewGitlabProvider(ctx context.Context, organization string, token string, url string, allBranches bool) (*GitlabProvider, error) { +func NewGitlabProvider(ctx context.Context, organization string, token string, url string, allBranches, includeSubgroups bool) (*GitlabProvider, error) { // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. if token == "" { token = os.Getenv("GITLAB_TOKEN") @@ -40,13 +36,13 @@ func NewGitlabProvider(ctx context.Context, organization string, token string, u return nil, err } } - return &GitlabProvider{client: client, organization: organization, allBranches: allBranches}, nil + return &GitlabProvider{client: client, organization: organization, allBranches: allBranches, includeSubgroups: includeSubgroups}, nil } func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([]*Repository, error) { opt := &gitlab.ListGroupProjectsOptions{ ListOptions: gitlab.ListOptions{PerPage: 100}, - IncludeSubgroups: newTrue(), + IncludeSubgroups: &g.includeSubgroups, } repos := []*Repository{} for { @@ -89,7 +85,7 @@ func (g *GitlabProvider) ListRepos(ctx context.Context, cloneProtocol string) ([ return repos, nil } -func (g *GitlabProvider) RepoHasPath(ctx context.Context, repo *Repository, path string) (bool, error) { +func (g *GitlabProvider) RepoHasPath(_ context.Context, repo *Repository, path string) (bool, error) { p, _, err := g.client.Projects.GetProject(repo.Organization+"/"+repo.Repository, nil) if err != nil { return false, err @@ -109,7 +105,7 @@ func (g *GitlabProvider) RepoHasPath(ctx context.Context, repo *Repository, path return true, nil } -func (g *GitlabProvider) listBranches(ctx context.Context, repo *gitlab.Project) ([]string, error) { +func (g *GitlabProvider) listBranches(_ context.Context, repo *gitlab.Project) ([]string, error) { // If we don't specifically want to query for all branches, just use the default branch and call it a day. if !g.allBranches { return []string{repo.DefaultBranch}, nil diff --git a/pkg/services/scm_provider/gitlab_test.go b/pkg/services/scm_provider/gitlab_test.go index 1c38dd5b..abb447f8 100644 --- a/pkg/services/scm_provider/gitlab_test.go +++ b/pkg/services/scm_provider/gitlab_test.go @@ -9,9 +9,9 @@ import ( func TestGitlabListRepos(t *testing.T) { cases := []struct { - name, proto, url string - hasError, allBranches bool - branches []string + name, proto, url string + hasError, allBranches, includeSubgroups bool + branches []string }{ { name: "blank protocol", @@ -43,7 +43,7 @@ func TestGitlabListRepos(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - provider, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", c.allBranches) + provider, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", c.allBranches, c.includeSubgroups) rawRepos, err := provider.ListRepos(context.Background(), c.proto) if c.hasError { assert.NotNil(t, err) @@ -70,7 +70,7 @@ func TestGitlabListRepos(t *testing.T) { } func TestGitlabHasPath(t *testing.T) { - host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", false) + host, _ := NewGitlabProvider(context.Background(), "test-argocd-proton", "", "", false, true) repo := &Repository{ Organization: "test-argocd-proton", Repository: "argocd", From cd732530aeb8fe153cfba3e78b062fcb475781d1 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Wed, 7 Jul 2021 10:16:19 -0400 Subject: [PATCH 10/14] removed print debugging --- pkg/services/scm_provider/gitlab.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/services/scm_provider/gitlab.go b/pkg/services/scm_provider/gitlab.go index 8db9b8d7..433abfde 100644 --- a/pkg/services/scm_provider/gitlab.go +++ b/pkg/services/scm_provider/gitlab.go @@ -94,8 +94,6 @@ func (g *GitlabProvider) RepoHasPath(_ context.Context, repo *Repository, path s Path: &path, Ref: &repo.Branch, }) - fmt.Printf("%+v\n", err) - fmt.Printf("%+v\n", resp) if resp.TotalItems == 0 { return false, nil } From 3cf2d262a28945f436c86ea9c2699c6dc7bf09e5 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Wed, 7 Jul 2021 10:21:41 -0400 Subject: [PATCH 11/14] updated docs --- docs/Generators-SCM-Provider.md | 40 ++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/Generators-SCM-Provider.md b/docs/Generators-SCM-Provider.md index e696d7b6..eefc48e1 100644 --- a/docs/Generators-SCM-Provider.md +++ b/docs/Generators-SCM-Provider.md @@ -2,7 +2,6 @@ The SCM Provider generator uses the API of an SCMaaS provider (eg GitHub) to automatically discover repositories within an organization. This fits well with GitOps layout patterns that split microservices across many repositories. -Support is currently limited to GitHub, but PRs are welcome to add more SCM providers. ```yaml apiVersion: argoproj.io/v1alpha1 @@ -57,6 +56,45 @@ For label filtering, the repository topics are used. Available clone protocols are `ssh` and `https`. +## Gitlab + +The Gitlab mode uses the Gitlab API to scan and organization in either gitlab.com or self-hosted gitlab. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapps +spec: + generators: + - scmProvider: + gitlab: + # The base Gitlab group to scan. You can either use the group id or the full namespaced path. + group: "8675309" + # For GitHub Enterprise: + api: https://gitlab.example.com/ + # If true, scan every branch of every repository. If false, scan only the default branch. Defaults to false. + allBranches: true + # If true, recurses through subgroups. If false, it searches only in the base group. Defaults to false. + includeSubgroups: true + # Reference to a Secret containing an access token. (optional) + tokenRef: + secretName: gitlab-token + key: token + template: + # ... +``` + +* `group`: Required name of the base Gitlab group to scan. If you have multiple base groups, use multiple generators. +* `api`: If using GitHub Enterprise, the URL to access it. +* `allBranches`: By default (false) the template will only be evaluated for the default branch of each repo. If this is true, every branch of every repository will be passed to the filters. If using this flag, you likely want to use a `branchMatch` filter. +* `allBranches`: By default (false) the controller will only search for repos directly in the base group. If this is true, it will recurse through all the subgroups searching for repos to scan. +* `tokenRef`: A `Secret` name and key containing the Gitlab access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories. + +For label filtering, the repository tags are used. + +Available clone protocols are `ssh` and `https`. + ## Filters Filters allow selecting which repositories to generate for. Each filter can declare one or more conditions, all of which must pass. If multiple filters are present, any can match for a repository to be included. If no filters are specified, all repositories will be processed. From cc9803c14b8ef7aab56570ca91f4ff5dd7809f38 Mon Sep 17 00:00:00 2001 From: empath-nirvana <86739375+empath-nirvana@users.noreply.github.com> Date: Wed, 7 Jul 2021 10:22:46 -0400 Subject: [PATCH 12/14] Update applicationset_types.go --- api/v1alpha1/applicationset_types.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index 7545aa3c..19a4dfc4 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -159,7 +159,6 @@ type SCMProviderGenerator struct { // Which provider to use and config for it. Github *SCMProviderGeneratorGithub `json:"github,omitempty"` Gitlab *SCMProviderGeneratorGitlab `json:"gitlab,omitempty"` - // TODO other providers. // Filters for which repos should be considered. Filters []SCMProviderGeneratorFilter `json:"filters,omitempty"` // Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers From e5446dd6192f027d8ef5ce2ccb8c196b72c7b61d Mon Sep 17 00:00:00 2001 From: John Thompson Date: Wed, 7 Jul 2021 10:23:52 -0400 Subject: [PATCH 13/14] updated manifests --- manifests/crds/argoproj.io_applicationsets.yaml | 17 +++++++++++++++-- manifests/install-with-argo-cd.yaml | 10 ++++++++-- manifests/install.yaml | 10 ++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 44f1ccf6..9073d73e 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -3817,8 +3817,15 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: Gitlab org to scan. Required. + description: Gitlab group to scan. Required. You + can use either the project id (recommended) + or the full namespaced path. type: string + includeSubgroups: + description: Recurse through subgroups (true) + or scan only the base group (false). Defaults + to "false" + type: boolean tokenRef: description: Authentication token reference. properties: @@ -4740,8 +4747,14 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: Gitlab org to scan. Required. + description: Gitlab group to scan. Required. You can + use either the project id (recommended) or the full + namespaced path. type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan + only the base group (false). Defaults to "false" + type: boolean tokenRef: description: Authentication token reference. properties: diff --git a/manifests/install-with-argo-cd.yaml b/manifests/install-with-argo-cd.yaml index a810ad82..8ff1d415 100644 --- a/manifests/install-with-argo-cd.yaml +++ b/manifests/install-with-argo-cd.yaml @@ -4679,8 +4679,11 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: Gitlab org to scan. Required. + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean tokenRef: description: Authentication token reference. properties: @@ -5398,8 +5401,11 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: Gitlab org to scan. Required. + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean tokenRef: description: Authentication token reference. properties: diff --git a/manifests/install.yaml b/manifests/install.yaml index 6bd801e9..1d33ff40 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2918,8 +2918,11 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: Gitlab org to scan. Required. + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean tokenRef: description: Authentication token reference. properties: @@ -3637,8 +3640,11 @@ spec: description: The Gitlab API URL to talk to. type: string group: - description: Gitlab org to scan. Required. + description: Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. type: string + includeSubgroups: + description: Recurse through subgroups (true) or scan only the base group (false). Defaults to "false" + type: boolean tokenRef: description: Authentication token reference. properties: From 74016d5cf69e4b4b5ff142adcb41d3fc2c70dece Mon Sep 17 00:00:00 2001 From: John Thompson Date: Wed, 7 Jul 2021 12:35:01 -0400 Subject: [PATCH 14/14] fixed manifests --- manifests/crds/argoproj.io_applicationsets.yaml | 7 +++---- manifests/install-with-argo-cd.yaml | 4 ++-- manifests/install.yaml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 9073d73e..4ed09000 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -3748,8 +3748,8 @@ spec: all protocols. type: string filters: - description: TODO other providers. Filters for - which repos should be considered. + description: Filters for which repos should be + considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter @@ -4681,8 +4681,7 @@ spec: necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos - should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single diff --git a/manifests/install-with-argo-cd.yaml b/manifests/install-with-argo-cd.yaml index 8ff1d415..38b34b39 100644 --- a/manifests/install-with-argo-cd.yaml +++ b/manifests/install-with-argo-cd.yaml @@ -4623,7 +4623,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: @@ -5345,7 +5345,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: diff --git a/manifests/install.yaml b/manifests/install.yaml index 1d33ff40..97876c85 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2862,7 +2862,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: @@ -3584,7 +3584,7 @@ spec: description: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols. type: string filters: - description: TODO other providers. Filters for which repos should be considered. + description: Filters for which repos should be considered. items: description: SCMProviderGeneratorFilter is a single repository filter. If multiple filter types are set on a single struct, they will be AND'd together. All filters must pass for a repo to be included. properties: