From 0fff564f6ef91e714dd384f2fec4ae95b1987968 Mon Sep 17 00:00:00 2001 From: WuTheFWasThat Date: Tue, 16 Jun 2026 21:30:32 +0700 Subject: [PATCH 1/2] feat(azuredevops_go): implement Azure DevOps Boards (TICKET domain) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add board/team scopes with work item collection, iteration/sprint support, cross-domain WI↔PR/commit links, and project-level deduplicated pipeline stages. Update Config UI for board selection and Grafana dashboards for issue metrics. Closes magiskboy/devlake#2 Closes magiskboy/devlake#3 Co-authored-by: Cursor --- backend/plugins/azuredevops_go/README.md | 51 +- .../azuredevops_go/api/azuredevops/models.go | 86 ++ .../api/azuredevops/wit_client.go | 378 +++++++ .../azuredevops_go/api/blueprint_v200.go | 363 ++++--- .../azuredevops_go/api/blueprint_v200_test.go | 345 +++---- backend/plugins/azuredevops_go/api/init.go | 28 +- .../azuredevops_go/api/remote_board_helper.go | 117 +++ ...remote_helper.go => remote_repo_helper.go} | 125 +-- .../plugins/azuredevops_go/api/repo_api.go | 56 ++ .../plugins/azuredevops_go/api/scope_api.go | 63 +- .../azuredevops_go/api/scope_config_api.go | 11 +- .../plugins/azuredevops_go/e2e/board_test.go | 52 + .../azuredevops_go/e2e/iteration_test.go | 68 ++ .../_tool_azuredevops_go_board_iterations.csv | 2 + .../_tool_azuredevops_go_board_work_items.csv | 2 + .../_tool_azuredevops_go_boards.csv | 2 + ...ol_azuredevops_go_iteration_work_items.csv | 2 + .../_tool_azuredevops_go_iterations.csv | 2 + ...uredevops_go_work_item_changelog_items.csv | 2 + ...ol_azuredevops_go_work_item_changelogs.csv | 2 + .../_tool_azuredevops_go_work_items.csv | 2 + .../e2e/snapshot_tables/board_issues.csv | 2 + .../e2e/snapshot_tables/board_sprints.csv | 2 + .../e2e/snapshot_tables/boards.csv | 2 + .../e2e/snapshot_tables/issue_changelogs.csv | 2 + .../e2e/snapshot_tables/issues.csv | 2 + .../e2e/snapshot_tables/sprint_issues.csv | 2 + .../e2e/snapshot_tables/sprints.csv | 2 + .../e2e/work_item_changelog_test.go | 53 + .../azuredevops_go/e2e/work_item_test.go | 61 ++ backend/plugins/azuredevops_go/impl/impl.go | 62 +- .../plugins/azuredevops_go/models/board.go | 69 ++ .../azuredevops_go/models/board_work_item.go | 33 + .../azuredevops_go/models/iteration.go | 63 ++ .../20260616_add_board_tables.go | 66 ++ .../models/migrationscripts/register.go | 2 + .../azuredevops_go/models/scope_config.go | 19 +- .../azuredevops_go/models/work_item.go | 55 + .../models/work_item_changelog.go | 55 + .../azuredevops_go/tasks/board_converter.go | 86 ++ .../tasks/board_work_item_extractor.go | 104 ++ .../azuredevops_go/tasks/commit_collector.go | 5 +- .../azuredevops_go/tasks/cross_domain_task.go | 247 +++++ .../tasks/iteration_converter.go | 102 ++ .../azuredevops_go/tasks/iteration_task.go | 239 +++++ .../azuredevops_go/tasks/pr_collector.go | 5 +- .../tasks/pr_commit_collector.go | 5 +- .../tasks/pr_commit_extractor.go | 2 +- .../azuredevops_go/tasks/pr_extractor.go | 2 +- .../azuredevops_go/tasks/repo_converter.go | 6 +- .../plugins/azuredevops_go/tasks/shared.go | 4 + .../plugins/azuredevops_go/tasks/task_data.go | 69 +- .../azuredevops_go/tasks/type_mapping.go | 114 +++ .../tasks/work_item_changelog_task.go | 236 +++++ .../tasks/work_item_collector.go | 181 ++++ .../tasks/work_item_converter.go | 129 +++ .../tasks/work_item_extractor.go | 142 +++ config-ui/src/api/scope/index.ts | 74 +- .../connection-form/fields/index.tsx | 12 +- .../components/connection-form/index.tsx | 4 +- .../components/connection-list/index.tsx | 2 +- .../data-scope-remote/data-scope-remote.tsx | 16 +- .../data-scope-remote/search-local.tsx | 1 + .../data-scope-remote/search-remote.tsx | 2 + .../components/data-scope-select/index.tsx | 88 +- .../src/plugins/register/azure/config.tsx | 38 +- .../azure/connection-fields/organization.tsx | 6 +- .../plugins/register/azure/transformation.tsx | 136 ++- config-ui/src/plugins/utils.ts | 2 + .../src/routes/connection/connection.tsx | 227 ++++- .../src/routes/connection/connections.tsx | 4 +- .../src/routes/pipeline/components/task.tsx | 2 +- config-ui/src/types/plugin.ts | 3 + docker-compose-dev-mysql.yml | 1 + grafana/dashboards/mysql/azure-dev-ops.json | 942 ++++++++++++++++++ .../dashboards/postgresql/azure-dev-ops.json | 942 ++++++++++++++++++ 76 files changed, 5905 insertions(+), 586 deletions(-) create mode 100644 backend/plugins/azuredevops_go/api/azuredevops/wit_client.go create mode 100644 backend/plugins/azuredevops_go/api/remote_board_helper.go rename backend/plugins/azuredevops_go/api/{remote_helper.go => remote_repo_helper.go} (71%) create mode 100644 backend/plugins/azuredevops_go/api/repo_api.go create mode 100644 backend/plugins/azuredevops_go/e2e/board_test.go create mode 100644 backend/plugins/azuredevops_go/e2e/iteration_test.go create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_iterations.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_work_items.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_boards.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iteration_work_items.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iterations.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelog_items.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelogs.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_items.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/board_issues.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/board_sprints.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/boards.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/issue_changelogs.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/issues.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/sprint_issues.csv create mode 100644 backend/plugins/azuredevops_go/e2e/snapshot_tables/sprints.csv create mode 100644 backend/plugins/azuredevops_go/e2e/work_item_changelog_test.go create mode 100644 backend/plugins/azuredevops_go/e2e/work_item_test.go create mode 100644 backend/plugins/azuredevops_go/models/board.go create mode 100644 backend/plugins/azuredevops_go/models/board_work_item.go create mode 100644 backend/plugins/azuredevops_go/models/iteration.go create mode 100644 backend/plugins/azuredevops_go/models/migrationscripts/20260616_add_board_tables.go create mode 100644 backend/plugins/azuredevops_go/models/work_item.go create mode 100644 backend/plugins/azuredevops_go/models/work_item_changelog.go create mode 100644 backend/plugins/azuredevops_go/tasks/board_converter.go create mode 100644 backend/plugins/azuredevops_go/tasks/board_work_item_extractor.go create mode 100644 backend/plugins/azuredevops_go/tasks/cross_domain_task.go create mode 100644 backend/plugins/azuredevops_go/tasks/iteration_converter.go create mode 100644 backend/plugins/azuredevops_go/tasks/iteration_task.go create mode 100644 backend/plugins/azuredevops_go/tasks/type_mapping.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_changelog_task.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_collector.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_converter.go create mode 100644 backend/plugins/azuredevops_go/tasks/work_item_extractor.go diff --git a/backend/plugins/azuredevops_go/README.md b/backend/plugins/azuredevops_go/README.md index 1ea487025df..5b58fa74727 100644 --- a/backend/plugins/azuredevops_go/README.md +++ b/backend/plugins/azuredevops_go/README.md @@ -15,16 +15,49 @@ See the License for the specific language governing permissions and limitations under the License. --> -# Azure Devops Python Plugin +# Azure DevOps Go Plugin -This is a revamped version of the Python Azure DevOps Plugin, originally located at `../../python/plugins/azuredevops`. -The plugin is able to coexist with the Python version as both implementations come with their own `_raw` and `_tool` tables. +Go implementation of the Azure DevOps plugin for Apache DevLake. Collects repository (CODE/CODEREVIEW/CROSS), pipeline (CICD), and board/work item (TICKET) data. -**Read access** to the following Azure DevOps Scopes is required: +## Authentication -- Build -- Code -- Graph (collectAccounts task) -- Release +Create a Personal Access Token (PAT) with the following scopes: -Access to Service Connections has been removed as they usually contain sensitive security information. \ No newline at end of file +| Scope | Purpose | +|-------|---------| +| **Code (read)** | Repositories, commits, pull requests | +| **Build (read)** | Pipelines and CI/CD runs | +| **Work Items (read)** (`vso.work`) | Boards, work items, iterations, changelogs | +| **Graph (read)** | User accounts (optional) | + +When creating the PAT, select **All accessible organizations** if you need multi-org support. + +## Data Scopes + +This plugin uses **dual scopes** on a single connection: + +| Scope type | API path | Domain entities | +|------------|----------|-----------------| +| **Repositories** | `/connections/:id/repos` | CODE, CODEREVIEW, CROSS, CICD | +| **Boards / Teams** | `/connections/:id/scopes` | TICKET | + +In Config UI, add repositories and board/team scopes separately. Associate a Scope Config with the entities you need (e.g. enable TICKET for board scopes). + +## Scope Config (TICKET) + +Configure work item type mappings in the Scope Config transformation panel: + +- **Requirement**: User Story, Product Backlog Item, Feature, Epic +- **Bug**: Bug +- **Incident**: Production Incident (maps to DORA incidents via `project_mapping`) +- **Story Points field**: defaults to `Microsoft.VSTS.Scheduling.StoryPoints` + +Sprint changes in changelogs are normalized to `field_name = 'Sprint'` for Engineering Overview dashboards. + +## Blueprint + +When adding scopes to a project blueprint, DevLake resolves each scope ID against boards first, then repositories. Board scopes run the TICKET pipeline independently; repository scopes no longer create a stub board. + +## Grafana + +The **Azure DevOps** dashboard includes Issue Throughput and Issue Lead Time panels when TICKET data is collected. Use the **Board / Team** filter for `azuredevops_go` boards. diff --git a/backend/plugins/azuredevops_go/api/azuredevops/models.go b/backend/plugins/azuredevops_go/api/azuredevops/models.go index 388d60cfc69..1c5444a953b 100644 --- a/backend/plugins/azuredevops_go/api/azuredevops/models.go +++ b/backend/plugins/azuredevops_go/api/azuredevops/models.go @@ -130,3 +130,89 @@ type Repository struct { IsDisabled bool `json:"isDisabled"` IsInMaintenance bool `json:"isInMaintenance"` } + +type Team struct { + Id string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Url string `json:"url"` +} + +type WiqlResponse struct { + QueryType string `json:"queryType"` + QueryResult struct { + WorkItems []struct { + Id int `json:"id"` + Url string `json:"url"` + } `json:"workItems"` + } `json:"queryResult"` + WorkItems []struct { + Id int `json:"id"` + Url string `json:"url"` + } `json:"workItems"` +} + +type WorkItemBatchRequest struct { + Ids []int `json:"ids"` + Fields []string `json:"fields,omitempty"` + Expand string `json:"$expand,omitempty"` + ErrorPolicy string `json:"errorPolicy,omitempty"` +} + +type WorkItem struct { + Id int `json:"id"` + Rev int `json:"rev"` + Fields map[string]interface{} `json:"fields"` + Url string `json:"url"` + Relations []WorkItemRelation `json:"relations"` +} + +type WorkItemRelation struct { + Rel string `json:"rel"` + Url string `json:"url"` + Attributes map[string]interface{} `json:"attributes"` +} + +type TeamIteration struct { + Id string `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + Attributes struct { + StartDate *time.Time `json:"startDate"` + FinishDate *time.Time `json:"finishDate"` + TimeFrame string `json:"timeFrame"` + } `json:"attributes"` +} + +type WorkItemUpdate struct { + Id int `json:"id"` + Rev int `json:"rev"` + Fields struct { + SystemChangedDate struct { + NewValue string `json:"newValue"` + } `json:"System.ChangedDate"` + } `json:"fields"` + Relations *struct { + Added []WorkItemRelation `json:"added"` + } `json:"relations"` +} + +type WorkItemUpdatesResponse struct { + Count int `json:"count"` + Value []WorkItemUpdate `json:"value"` +} + +type IterationWorkItemsResponse struct { + WorkItemRelations []struct { + Rel string `json:"rel"` + Source struct { + Id int `json:"id"` + Url string `json:"url"` + } `json:"source"` + Target struct { + Id int `json:"id"` + Url string `json:"url"` + } `json:"target"` + } `json:"workItemRelations"` +} + diff --git a/backend/plugins/azuredevops_go/api/azuredevops/wit_client.go b/backend/plugins/azuredevops_go/api/azuredevops/wit_client.go new file mode 100644 index 00000000000..675f249e357 --- /dev/null +++ b/backend/plugins/azuredevops_go/api/azuredevops/wit_client.go @@ -0,0 +1,378 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azuredevops + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/url" + "strconv" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const workItemBatchSize = 200 + +var defaultWorkItemFields = []string{ + "System.Id", + "System.Title", + "System.WorkItemType", + "System.State", + "System.CreatedDate", + "System.ChangedDate", + "Microsoft.VSTS.Common.ClosedDate", + "Microsoft.VSTS.Common.Priority", + "Microsoft.VSTS.Scheduling.StoryPoints", + "System.IterationPath", + "System.AssignedTo", + "System.CreatedBy", + "System.Parent", +} + +type GetTeamsArgs struct { + OrgId string + ProjectId string +} + +func (c *Client) GetTeams(args GetTeamsArgs) ([]Team, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + + path := fmt.Sprintf("%s/_apis/projects/%s/teams", args.OrgId, args.ProjectId) + res, err := c.apiClient.Get(path, query, nil) + if err != nil { + return nil, err + } + if res.StatusCode == 401 || res.StatusCode == 403 { + return nil, errors.Unauthorized.New("failed to authorize teams request") + } + + var data struct { + Value []Team `json:"value"` + } + if err = api.UnmarshalResponse(res, &data); err != nil { + return nil, err + } + return data.Value, nil +} + +type QueryWorkItemsArgs struct { + OrgId string + ProjectId string + TeamId string + Wiql string +} + +func (c *Client) QueryWorkItems(args QueryWorkItemsArgs) ([]int, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + + body := map[string]string{"query": args.Wiql} + path := fmt.Sprintf("%s/%s/%s/_apis/wit/wiql", args.OrgId, args.ProjectId, args.TeamId) + res, err := c.apiClient.Post(path, query, body, nil) + if err != nil { + return nil, err + } + if res.StatusCode == 401 || res.StatusCode == 403 { + return nil, errors.Unauthorized.New("failed to authorize wiql request") + } + + var data WiqlResponse + if err = api.UnmarshalResponse(res, &data); err != nil { + return nil, err + } + + items := data.WorkItems + if len(items) == 0 { + items = data.QueryResult.WorkItems + } + ids := make([]int, 0, len(items)) + for _, wi := range items { + ids = append(ids, wi.Id) + } + return ids, nil +} + +type GetWorkItemsBatchArgs struct { + OrgId string + ProjectId string + Ids []int + Fields []string + Expand string +} + +func (c *Client) GetWorkItemsBatch(args GetWorkItemsBatchArgs) ([]WorkItem, errors.Error) { + if len(args.Ids) == 0 { + return nil, nil + } + fields := args.Fields + if len(fields) == 0 { + fields = defaultWorkItemFields + } + // Azure DevOps batch API rejects request when "$expand" and "fields" + // are used together. For relation-expansion mode, omit fields. + if args.Expand != "" { + fields = nil + } + + query := url.Values{} + query.Set("api-version", apiVersion) + + var allItems []WorkItem + for i := 0; i < len(args.Ids); i += workItemBatchSize { + end := i + workItemBatchSize + if end > len(args.Ids) { + end = len(args.Ids) + } + batch := args.Ids[i:end] + req := WorkItemBatchRequest{ + Ids: batch, + Fields: fields, + Expand: args.Expand, + ErrorPolicy: "Omit", + } + path := fmt.Sprintf("%s/_apis/wit/workitemsbatch", args.OrgId) + if args.ProjectId != "" { + path = fmt.Sprintf("%s/%s/_apis/wit/workitemsbatch", args.OrgId, args.ProjectId) + } + res, err := c.apiClient.Post(path, query, req, nil) + if err != nil { + return nil, err + } + switch res.StatusCode { + case 401, 403: + _ = res.Body.Close() + return nil, errors.Unauthorized.New("failed to authorize work items batch request") + case 200: + default: + body, readErr := io.ReadAll(res.Body) + _ = res.Body.Close() + if readErr != nil { + return nil, errors.Convert(readErr) + } + return nil, errors.HttpStatus(res.StatusCode). + New(fmt.Sprintf("work items batch request failed with status %d, response: %s", res.StatusCode, string(body))) + } + var data struct { + Value []WorkItem `json:"value"` + } + if err = api.UnmarshalResponse(res, &data); err != nil { + return nil, err + } + allItems = append(allItems, data.Value...) + } + return allItems, nil +} + +type GetTeamIterationsArgs struct { + OrgId string + ProjectId string + TeamId string +} + +func (c *Client) GetTeamIterations(args GetTeamIterationsArgs) ([]TeamIteration, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + + path := fmt.Sprintf("%s/%s/%s/_apis/work/teamsettings/iterations", args.OrgId, args.ProjectId, args.TeamId) + res, err := c.apiClient.Get(path, query, nil) + if err != nil { + return nil, err + } + + var data struct { + Value []TeamIteration `json:"value"` + } + if err = api.UnmarshalResponse(res, &data); err != nil { + return nil, err + } + return data.Value, nil +} + +type GetIterationWorkItemsArgs struct { + OrgId string + ProjectId string + TeamId string + IterationId string +} + +func (c *Client) GetIterationWorkItems(args GetIterationWorkItemsArgs) ([]int, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + + path := fmt.Sprintf("%s/%s/%s/_apis/work/teamsettings/iterations/%s/workitems", + args.OrgId, args.ProjectId, args.TeamId, args.IterationId) + res, err := c.apiClient.Get(path, query, nil) + if err != nil { + return nil, err + } + + var data IterationWorkItemsResponse + if err = api.UnmarshalResponse(res, &data); err != nil { + return nil, err + } + + ids := make([]int, 0) + for _, rel := range data.WorkItemRelations { + if rel.Target.Id != 0 { + ids = append(ids, rel.Target.Id) + } + } + return ids, nil +} + +type GetWorkItemUpdatesArgs struct { + OrgId string + WorkItemId int +} + +func (c *Client) GetWorkItemUpdates(args GetWorkItemUpdatesArgs) ([]WorkItemUpdate, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + + path := fmt.Sprintf("%s/_apis/wit/workitems/%d/updates", args.OrgId, args.WorkItemId) + res, err := c.apiClient.Get(path, query, nil) + if err != nil { + return nil, err + } + + var data WorkItemUpdatesResponse + if err = api.UnmarshalResponse(res, &data); err != nil { + return nil, err + } + return data.Value, nil +} + +type GetWorkItemWithRelationsArgs struct { + OrgId string + WorkItemId int +} + +func (c *Client) GetWorkItemWithRelations(args GetWorkItemWithRelationsArgs) (*WorkItem, errors.Error) { + query := url.Values{} + query.Set("api-version", apiVersion) + query.Set("$expand", "relations") + + path := fmt.Sprintf("%s/_apis/wit/workitems/%d", args.OrgId, args.WorkItemId) + res, err := c.apiClient.Get(path, query, nil) + if err != nil { + return nil, err + } + + var item WorkItem + if err = api.UnmarshalResponse(res, &item); err != nil { + return nil, err + } + return &item, nil +} + +// BuildWiqlQuery builds an incremental WIQL query for work items in a project. +func BuildWiqlQuery(projectName string, since string) string { + if since == "" { + return fmt.Sprintf( + "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = '%s' ORDER BY [System.ChangedDate] ASC", + escapeWiql(projectName), + ) + } + return fmt.Sprintf( + "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = '%s' AND [System.ChangedDate] >= '%s' ORDER BY [System.ChangedDate] ASC", + escapeWiql(projectName), + since, + ) +} + +func escapeWiql(s string) string { + return string(bytes.ReplaceAll([]byte(s), []byte("'"), []byte("''"))) +} + +// FieldString extracts a string field from a work item. +func FieldString(fields map[string]interface{}, key string) string { + v, ok := fields[key] + if !ok || v == nil { + return "" + } + switch val := v.(type) { + case string: + return val + case map[string]interface{}: + if display, ok := val["displayName"].(string); ok { + return display + } + if unique, ok := val["uniqueName"].(string); ok { + return unique + } + } + return fmt.Sprintf("%v", v) +} + +// FieldIdentityId extracts identity id from an identity field. +func FieldIdentityId(fields map[string]interface{}, key string) string { + v, ok := fields[key] + if !ok || v == nil { + return "" + } + if m, ok := v.(map[string]interface{}); ok { + if id, ok := m["id"].(string); ok { + return id + } + } + return "" +} + +// FieldFloat extracts a float field from a work item. +func FieldFloat(fields map[string]interface{}, key string) *float64 { + v, ok := fields[key] + if !ok || v == nil { + return nil + } + switch val := v.(type) { + case float64: + return &val + case json.Number: + f, err := val.Float64() + if err != nil { + return nil + } + return &f + case string: + f, err := strconv.ParseFloat(val, 64) + if err != nil { + return nil + } + return &f + } + return nil +} + +// FieldInt extracts parent work item id. +func FieldInt(fields map[string]interface{}, key string) int { + v, ok := fields[key] + if !ok || v == nil { + return 0 + } + switch val := v.(type) { + case float64: + return int(val) + case int: + return val + } + return 0 +} diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go index 018d9d1d77e..eb918a9099b 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go @@ -32,8 +32,8 @@ import ( "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/helpers/srvhelper" "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/tasks" ) func MakePipelinePlanV200( @@ -41,169 +41,278 @@ func MakePipelinePlanV200( connectionId uint64, bpScopes []*coreModels.BlueprintScope, ) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { - // load connection, scope and scopeConfig from the db connection, err := dsHelper.ConnSrv.FindByPk(connectionId) if err != nil { return nil, nil, err } - scopeDetails, err := dsHelper.ScopeApi.MapScopeDetails(connectionId, bpScopes) - if err != nil { - return nil, nil, err - } - sc, err := makeScopeV200(connectionId, scopeDetails) - if err != nil { - return nil, nil, err - } + plans := make(coreModels.PipelinePlan, 0, 2*len(bpScopes)) + domainScopes := make([]plugin.Scope, 0, 2*len(bpScopes)) - pp, err := makePipelinePlanV200(subtaskMetas, connection, scopeDetails) - if err != nil { - return nil, nil, err + // Resolve all scopes in one pass, track unique projects from board scopes. + type boardEntry struct { + board *models.AzuredevopsBoard + config *models.AzuredevopsScopeConfig } - - return pp, sc, nil -} - -func makeScopeV200( - connectionId uint64, - scopeDetails []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig], -) ([]plugin.Scope, errors.Error) { - sc := make([]plugin.Scope, 0, 3*len(scopeDetails)) - - for _, scope := range scopeDetails { - azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - if azuredevopsRepo.Type != models.RepositoryTypeADO { + type repoEntry struct { + repo *models.AzuredevopsRepo + config *models.AzuredevopsScopeConfig + } + var boardEntries []boardEntry + var repoEntries []repoEntry + + // deduplicate project stages: key = "orgId/projectId" + seenProjects := map[string]bool{} + var projectStages coreModels.PipelinePlan + + for _, bpScope := range bpScopes { + boardDetail, boardErr := dsHelper.ScopeSrv.GetScopeDetail(false, connectionId, bpScope.ScopeId) + if boardErr == nil { + sc := boardDetail.ScopeConfig + if sc == nil { + sc = &models.AzuredevopsScopeConfig{} + } + if len(sc.Entities) == 0 { + sc.Entities = plugin.DOMAIN_TYPES + } + boardEntries = append(boardEntries, boardEntry{&boardDetail.Scope, sc}) + + // Emit one project-level WI stage per unique (org, project) pair + if hasTicketOrCrossEntity(sc.Entities) { + projKey := boardDetail.Scope.OrganizationId + "/" + boardDetail.Scope.ProjectId + if !seenProjects[projKey] { + seenProjects[projKey] = true + stage := makeProjectWorkItemStage(connection, &boardDetail.Scope, sc) + projectStages = append(projectStages, stage) + } + } continue } - id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id) - // if no entities specified, use all entities enabled by default - if len(scopeConfig.Entities) == 0 { - scopeConfig.Entities = plugin.DOMAIN_TYPES + repoDetail, repoErr := repoDsHelper.ScopeSrv.GetScopeDetail(false, connectionId, bpScope.ScopeId) + if repoErr != nil { + if boardErr != nil { + return nil, nil, boardErr + } + return nil, nil, repoErr } - - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) || - utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) || - utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CROSS) { - // if we don't need to collect gitex, we need to add repo to scopes here - scopeRepo := code.NewRepo(id, azuredevopsRepo.Name) - sc = append(sc, scopeRepo) + sc := repoDetail.ScopeConfig + if sc == nil { + sc = &models.AzuredevopsScopeConfig{} } - - // add cicd_scope to scopes - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) { - scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name) - sc = append(sc, scopeCICD) + if len(sc.Entities) == 0 { + sc.Entities = plugin.DOMAIN_TYPES } + repoEntries = append(repoEntries, repoEntry{&repoDetail.Scope, sc}) + } - // add board to scopes - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { - scopeTicket := ticket.NewBoard(id, azuredevopsRepo.Name) - sc = append(sc, scopeTicket) + // Project WI stages run first (one per unique project) + plans = append(plans, projectStages...) + + // Board stages — collectWorkItems / extractWorkItems are excluded here; + // they run exclusively in the project stage above. + boardSubtaskMetas := filterSubtaskMetasByName(subtaskMetas, + tasks.CollectWorkItemsMeta.Name, + tasks.ExtractWorkItemsMeta.Name, + ) + for _, be := range boardEntries { + stage, scopes, planErr := makeBoardPipelinePlan(boardSubtaskMetas, connection, be.board, be.config) + if planErr != nil { + return nil, nil, planErr } + plans = append(plans, stage) + domainScopes = append(domainScopes, scopes...) } - for _, scope := range scopeDetails { - azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - if azuredevopsRepo.Type == models.RepositoryTypeADO { - continue + // Repo stages: exclude project-level work item and board-only cross-domain subtasks. + repoSubtaskMetas := filterSubtaskMetasByName(subtaskMetas, + tasks.CollectWorkItemsMeta.Name, + tasks.ExtractWorkItemsMeta.Name, + tasks.ConvertIssueCommitsMeta.Name, + tasks.ConvertPullRequestIssuesMeta.Name, + tasks.ConvertIssueRepoCommitsMeta.Name, + ) + for _, re := range repoEntries { + stages, scopes, planErr := makeRepoPipelinePlan(repoSubtaskMetas, connection, re.repo, re.config) + if planErr != nil { + return nil, nil, planErr } - id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id) + plans = append(plans, stages...) + domainScopes = append(domainScopes, scopes...) + } - // Azure DevOps Pipeline can be used with remote repositories such as GitHub and Bitbucket - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) { - scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name) - sc = append(sc, scopeCICD) + return plans, domainScopes, nil +} + +// makeProjectWorkItemStage creates a pipeline stage that collects and extracts all work items +// for the given project exactly once, scoped to (connectionId, organizationId, projectId). +func makeProjectWorkItemStage( + connection *models.AzuredevopsConnection, + board *models.AzuredevopsBoard, + scopeConfig *models.AzuredevopsScopeConfig, +) coreModels.PipelineStage { + options := map[string]interface{}{ + "name": board.ProjectId, + "connectionId": connection.ID, + "organizationId": board.OrganizationId, + "projectId": board.ProjectId, + "scopeKind": tasks.ScopeKindProject, + "scopeConfig": scopeConfig, + } + return coreModels.PipelineStage{&coreModels.PipelineTask{ + Plugin: "azuredevops_go", + Subtasks: []string{tasks.CollectWorkItemsMeta.Name, tasks.ExtractWorkItemsMeta.Name}, + Options: options, + }} +} + +// hasTicketOrCrossEntity returns true when entities includes TICKET or CROSS domain types. +func hasTicketOrCrossEntity(entities []string) bool { + for _, e := range entities { + if e == plugin.DOMAIN_TYPE_TICKET || e == plugin.DOMAIN_TYPE_CROSS { + return true } + } + return false +} - // DOMAIN_TYPE_CODE (i.e. gitextractor, rediff) only works if the repository is public - if !azuredevopsRepo.IsPrivate && utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) { - scopeRepo := code.NewRepo(id, azuredevopsRepo.Name) - sc = append(sc, scopeRepo) +// filterSubtaskMetasByName returns a copy of metas with the named tasks removed. +func filterSubtaskMetasByName(metas []plugin.SubTaskMeta, excludeNames ...string) []plugin.SubTaskMeta { + excluded := make(map[string]bool, len(excludeNames)) + for _, n := range excludeNames { + excluded[n] = true + } + result := make([]plugin.SubTaskMeta, 0, len(metas)) + for _, m := range metas { + if !excluded[m.Name] { + result = append(result, m) } } + return result +} - return sc, nil +func makeBoardPipelinePlan( + subtaskMetas []plugin.SubTaskMeta, + connection *models.AzuredevopsConnection, + board *models.AzuredevopsBoard, + scopeConfig *models.AzuredevopsScopeConfig, +) (coreModels.PipelineStage, []plugin.Scope, errors.Error) { + var entities []string + for _, e := range scopeConfig.Entities { + if e == plugin.DOMAIN_TYPE_TICKET || e == plugin.DOMAIN_TYPE_CROSS { + entities = append(entities, e) + } + } + subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities) + if err != nil { + return nil, nil, err + } + options := map[string]interface{}{ + "name": board.Name, + "connectionId": connection.ID, + "organizationId": board.OrganizationId, + "projectId": board.ProjectId, + "teamId": board.TeamId, + "scopeKind": "board", + } + boardId := didgen.NewDomainIdGenerator(&models.AzuredevopsBoard{}).Generate(connection.ID, board.TeamId) + domainScopes := []plugin.Scope{} + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + domainScopes = append(domainScopes, ticket.NewBoard(boardId, board.ScopeFullName())) + } + stage := coreModels.PipelineStage{&coreModels.PipelineTask{ + Plugin: "azuredevops_go", + Subtasks: subtasks, + Options: options, + }} + return stage, domainScopes, nil } -func makePipelinePlanV200( +func makeRepoPipelinePlan( subtaskMetas []plugin.SubTaskMeta, connection *models.AzuredevopsConnection, - scopeDetails []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig], -) (coreModels.PipelinePlan, errors.Error) { - plans := make(coreModels.PipelinePlan, 0, 3*len(scopeDetails)) - for _, scope := range scopeDetails { - azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - var stage coreModels.PipelineStage - var err errors.Error - - options := make(map[string]interface{}) - options["name"] = azuredevopsRepo.Name // this is solely for the FE to display the repo name of a task - - options["connectionId"] = connection.ID - options["organizationId"] = azuredevopsRepo.OrganizationId - options["projectId"] = azuredevopsRepo.ProjectId - options["externalId"] = azuredevopsRepo.ExternalId - options["repositoryId"] = azuredevopsRepo.Id - options["repositoryType"] = azuredevopsRepo.Type - - // construct subtasks - var entities []string - if scope.Scope.Type == models.RepositoryTypeADO { - entities = append(entities, scopeConfig.Entities...) - } else { - if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD); i >= 0 { - entities = append(entities, scopeConfig.Entities[i]) - } + azuredevopsRepo *models.AzuredevopsRepo, + scopeConfig *models.AzuredevopsScopeConfig, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + var stages coreModels.PipelinePlan + domainScopes := make([]plugin.Scope, 0, 2) + + options := map[string]interface{}{ + "name": azuredevopsRepo.Name, + "connectionId": connection.ID, + "organizationId": azuredevopsRepo.OrganizationId, + "projectId": azuredevopsRepo.ProjectId, + "externalId": azuredevopsRepo.ExternalId, + "repositoryId": azuredevopsRepo.Id, + "repositoryType": azuredevopsRepo.Type, + "scopeKind": "repo", + } - if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE); i >= 0 && !scope.Scope.IsPrivate { - entities = append(entities, scopeConfig.Entities[i]) + var entities []string + if azuredevopsRepo.Type == models.RepositoryTypeADO { + for _, e := range scopeConfig.Entities { + if e != plugin.DOMAIN_TYPE_TICKET { + entities = append(entities, e) } } - - subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities) - if err != nil { - return nil, err + } else { + if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD); i >= 0 { + entities = append(entities, scopeConfig.Entities[i]) + } + if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE); i >= 0 && !azuredevopsRepo.IsPrivate { + entities = append(entities, scopeConfig.Entities[i]) } + } - stage = append(stage, &coreModels.PipelineTask{ - Plugin: "azuredevops_go", - Subtasks: subtasks, - Options: options, - }) + subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities) + if err != nil { + return nil, nil, err + } - // collect git data by gitextractor if CODE was requested - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) && !scope.Scope.IsPrivate || len(scopeConfig.Entities) == 0 { - cloneUrl, err := errors.Convert01(url.Parse(azuredevopsRepo.RemoteUrl)) - if err != nil { - return nil, err - } + stage := coreModels.PipelineStage{&coreModels.PipelineTask{ + Plugin: "azuredevops_go", + Subtasks: subtasks, + Options: options, + }} - if scope.Scope.Type == models.RepositoryTypeADO { - cloneUrl.User = url.UserPassword("git", connection.Token) - } - stage = append(stage, &coreModels.PipelineTask{ - Plugin: "gitextractor", - Options: map[string]interface{}{ - "url": cloneUrl.String(), - "name": azuredevopsRepo.Name, - "repoId": didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, azuredevopsRepo.Id), - "proxy": connection.Proxy, - "noShallowClone": true, - }, - }) + id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, azuredevopsRepo.Id) + if azuredevopsRepo.Type == models.RepositoryTypeADO || !azuredevopsRepo.IsPrivate { + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) || + utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) || + utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CROSS) { + domainScopes = append(domainScopes, code.NewRepo(id, azuredevopsRepo.Name)) } + } + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) { + domainScopes = append(domainScopes, devops.NewCicdScope(id, azuredevopsRepo.Name)) + } - plans = append(plans, stage) - - // refdiff part - if scopeConfig.Refdiff != nil { - task := &coreModels.PipelineTask{ - Plugin: "refdiff", - Options: scopeConfig.Refdiff, - } - plans = append(plans, coreModels.PipelineStage{task}) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) && !azuredevopsRepo.IsPrivate || len(scopeConfig.Entities) == 0 { + cloneUrl, err := errors.Convert01(url.Parse(azuredevopsRepo.RemoteUrl)) + if err != nil { + return nil, nil, err + } + if azuredevopsRepo.Type == models.RepositoryTypeADO { + cloneUrl.User = url.UserPassword("git", connection.Token) } + stage = append(stage, &coreModels.PipelineTask{ + Plugin: "gitextractor", + Options: map[string]interface{}{ + "url": cloneUrl.String(), + "name": azuredevopsRepo.Name, + "repoId": id, + "proxy": connection.Proxy, + "noShallowClone": true, + }, + }) + } + + stages = append(stages, stage) + if scopeConfig.Refdiff != nil { + stages = append(stages, coreModels.PipelineStage{&coreModels.PipelineTask{ + Plugin: "refdiff", + Options: scopeConfig.Refdiff, + }}) } - return plans, nil + return stages, domainScopes, nil } diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go index fd40353f7a8..2416a696858 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go @@ -24,121 +24,77 @@ import ( "testing" coreModels "github.com/apache/incubator-devlake/core/models" - mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin" - "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/helpers/srvhelper" "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" "github.com/apache/incubator-devlake/plugins/azuredevops_go/tasks" "github.com/stretchr/testify/assert" ) -const ( - connectionID uint64 = 1 - azuredevopsRepoId = "ad05901f-c9b0-4938-bc8a-a22eb2467ceb" - expectDomainScopeId = "azuredevops_go:AzuredevopsRepo:1:ad05901f-c9b0-4938-bc8a-a22eb2467ceb" -) - -func mockAzuredevopsPlugin(t *testing.T) { - mockMeta := mockplugin.NewPluginMeta(t) - mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/azuredevops_go") - mockMeta.On("Name").Return("dummy").Maybe() - err := plugin.RegisterPlugin("azuredevops_go", mockMeta) - assert.Equal(t, err, nil) -} +type stubAzurePlugin struct{} -func TestMakeScopes(t *testing.T) { - mockAzuredevopsPlugin(t) +func (stubAzurePlugin) RootPkgPath() string { return "github.com/apache/incubator-devlake/plugins/azuredevops_go" } +func (stubAzurePlugin) Name() string { return "azuredevops_go" } +func (stubAzurePlugin) Description() string { return "stub" } - actualScopes, err := makeScopeV200( - connectionID, - []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ - { - Scope: models.AzuredevopsRepo{ - Scope: common.Scope{ - ConnectionId: connectionID, - }, - Id: azuredevopsRepoId, - Type: models.RepositoryTypeADO, - }, - ScopeConfig: &models.AzuredevopsScopeConfig{ - ScopeConfig: common.ScopeConfig{ - Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD}, - }, - }, - }, - }, - ) - assert.Nil(t, err) - assert.Equal(t, 3, len(actualScopes)) - assert.Equal(t, actualScopes[0].ScopeId(), expectDomainScopeId) - assert.Equal(t, actualScopes[1].ScopeId(), expectDomainScopeId) - assert.Equal(t, actualScopes[2].ScopeId(), expectDomainScopeId) +func TestMain(m *testing.M) { + _ = plugin.RegisterPlugin("azuredevops_go", stubAzurePlugin{}) + m.Run() } -func TestMakeScopesWithEmptyEntities(t *testing.T) { - mockAzuredevopsPlugin(t) +const ( + connectionID uint64 = 1 + azuredevopsRepoId = "ad05901f-c9b0-4938-bc8a-a22eb2467ceb" + azuredevopsTeamId = "team-guid-123" + expectDomainScopeId = "azuredevops_go:AzuredevopsRepo:1:ad05901f-c9b0-4938-bc8a-a22eb2467ceb" + expectBoardDomainId = "azuredevops_go:AzuredevopsBoard:1:team-guid-123" +) - actualScopes, err := makeScopeV200( - connectionID, - []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ - { - Scope: models.AzuredevopsRepo{ - Scope: common.Scope{ - ConnectionId: connectionID, - }, - Id: azuredevopsRepoId, - Type: models.RepositoryTypeADO, - }, - ScopeConfig: &models.AzuredevopsScopeConfig{ - ScopeConfig: common.ScopeConfig{ - Entities: []string{}, - }, - }, +func TestMakeRepoScopes(t *testing.T) { + _, scopes, err := makeRepoPipelinePlan( + nil, + &models.AzuredevopsConnection{BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}}, + &models.AzuredevopsRepo{ + Scope: common.Scope{ConnectionId: connectionID}, + Id: azuredevopsRepoId, + Type: models.RepositoryTypeADO, + Name: "repo", + }, + &models.AzuredevopsScopeConfig{ + ScopeConfig: common.ScopeConfig{ + Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CICD}, }, }, ) assert.Nil(t, err) - // empty entities should default to all domain types, producing repo + cicd + board scopes - assert.Equal(t, 3, len(actualScopes)) - assert.Equal(t, actualScopes[0].ScopeId(), expectDomainScopeId) + assert.Equal(t, 2, len(scopes)) + assert.Equal(t, expectDomainScopeId, scopes[0].ScopeId()) + assert.NotEqual(t, "boards", scopes[0].TableName()) } -func TestMakeScopesWithCrossEntity(t *testing.T) { - mockAzuredevopsPlugin(t) - - actualScopes, err := makeScopeV200( - connectionID, - []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ - { - Scope: models.AzuredevopsRepo{ - Scope: common.Scope{ - ConnectionId: connectionID, - }, - Id: azuredevopsRepoId, - Type: models.RepositoryTypeADO, - }, - ScopeConfig: &models.AzuredevopsScopeConfig{ - ScopeConfig: common.ScopeConfig{ - Entities: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_TICKET}, - }, - }, +func TestMakeBoardScopes(t *testing.T) { + _, scopes, err := makeBoardPipelinePlan( + nil, + &models.AzuredevopsConnection{BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}}, + &models.AzuredevopsBoard{ + Scope: common.Scope{ConnectionId: connectionID}, + TeamId: azuredevopsTeamId, + Name: "Default Team", + }, + &models.AzuredevopsScopeConfig{ + ScopeConfig: common.ScopeConfig{ + Entities: []string{plugin.DOMAIN_TYPE_TICKET}, }, }, ) assert.Nil(t, err) - // CROSS entity should trigger repo scope creation, plus ticket = board scope - assert.Equal(t, 2, len(actualScopes)) - assert.Equal(t, actualScopes[0].ScopeId(), expectDomainScopeId) - assert.Equal(t, "repos", actualScopes[0].TableName()) - assert.Equal(t, "boards", actualScopes[1].TableName()) + assert.Equal(t, 1, len(scopes)) + assert.Equal(t, expectBoardDomainId, scopes[0].ScopeId()) + assert.Equal(t, "boards", scopes[0].TableName()) } -func TestMakeDataSourcePipelinePlanV200(t *testing.T) { - mockAzuredevopsPlugin(t) - +func TestMakeRepoPipelinePlan(t *testing.T) { const ( httpUrlToRepo = "https://this_is_cloneUrl" azureDevOpsToken = "personal-access-token" @@ -146,7 +102,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) { azureDevOpsOrgName = "azuredevops-test-org" ) - actualPlans, err := makePipelinePlanV200( + actualPlans, _, err := makeRepoPipelinePlan( []plugin.SubTaskMeta{ tasks.CollectApiPullRequestsMeta, tasks.ExtractApiPullRequestsMeta, @@ -154,146 +110,139 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) { tasks.ExtractApiBuildsMeta, }, &models.AzuredevopsConnection{ - BaseConnection: api.BaseConnection{ - Model: common.Model{ - ID: connectionID, - }, - }, + BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}, AzuredevopsConn: models.AzuredevopsConn{ - AzuredevopsAccessToken: models.AzuredevopsAccessToken{ - Token: azureDevOpsToken, - }, + AzuredevopsAccessToken: models.AzuredevopsAccessToken{Token: azureDevOpsToken}, }, }, - []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ - { - Scope: models.AzuredevopsRepo{ - Id: fmt.Sprint(azuredevopsRepoId), - AzureDevOpsPK: models.AzureDevOpsPK{ - ProjectId: azureDevOpsProjectName, - OrganizationId: azureDevOpsOrgName, - }, - Name: azureDevOpsProjectName, - Url: httpUrlToRepo, - RemoteUrl: httpUrlToRepo, - Type: models.RepositoryTypeADO, - }, - ScopeConfig: &models.AzuredevopsScopeConfig{ - ScopeConfig: common.ScopeConfig{ - Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CICD}, - }, - DeploymentPattern: "(?i)deploy", - ProductionPattern: "(?i)prod", - Refdiff: map[string]interface{}{ - "tagsPattern": "pattern", - "tagsLimit": 10, - "tagsOrder": "reverse semver", - }, - }, + &models.AzuredevopsRepo{ + Id: azuredevopsRepoId, + AzureDevOpsPK: models.AzureDevOpsPK{ + ProjectId: azureDevOpsProjectName, OrganizationId: azureDevOpsOrgName, }, + Name: azureDevOpsProjectName, Url: httpUrlToRepo, RemoteUrl: httpUrlToRepo, + Type: models.RepositoryTypeADO, + }, + &models.AzuredevopsScopeConfig{ + ScopeConfig: common.ScopeConfig{ + Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CICD}, + }, + Refdiff: map[string]interface{}{"tagsPattern": "pattern", "tagsLimit": 10, "tagsOrder": "reverse semver"}, }, ) assert.Nil(t, err) - var expectPlans = coreModels.PipelinePlan{ + expectPlans := coreModels.PipelinePlan{ { - { - Plugin: "azuredevops_go", - Subtasks: []string{ - tasks.CollectApiPullRequestsMeta.Name, - tasks.ExtractApiPullRequestsMeta.Name, - tasks.CollectBuildsMeta.Name, - tasks.ExtractApiBuildsMeta.Name, - }, - Options: map[string]interface{}{ - "name": azureDevOpsProjectName, - "connectionId": connectionID, - "projectId": azureDevOpsProjectName, - "repositoryId": fmt.Sprint(azuredevopsRepoId), - "organizationId": azureDevOpsOrgName, - "repositoryType": models.RepositoryTypeADO, - "externalId": "", - }, - }, - { - Plugin: "gitextractor", - Options: map[string]interface{}{ - "proxy": "", - "repoId": expectDomainScopeId, - "name": azureDevOpsProjectName, - "url": "https://git:personal-access-token@this_is_cloneUrl", - "noShallowClone": true, - }, - }, + {Plugin: "azuredevops_go", Subtasks: []string{ + tasks.CollectApiPullRequestsMeta.Name, tasks.ExtractApiPullRequestsMeta.Name, + tasks.CollectBuildsMeta.Name, tasks.ExtractApiBuildsMeta.Name, + }, Options: map[string]interface{}{ + "name": azureDevOpsProjectName, "connectionId": connectionID, + "projectId": azureDevOpsProjectName, "repositoryId": fmt.Sprint(azuredevopsRepoId), + "organizationId": azureDevOpsOrgName, "repositoryType": models.RepositoryTypeADO, + "externalId": "", "scopeKind": "repo", + }}, + {Plugin: "gitextractor", Options: map[string]interface{}{ + "proxy": "", "repoId": expectDomainScopeId, "name": azureDevOpsProjectName, + "url": "https://git:personal-access-token@this_is_cloneUrl", "noShallowClone": true, + }}, }, - { - { - Plugin: "refdiff", - Options: map[string]interface{}{ - "tagsLimit": 10, - "tagsOrder": "reverse semver", - "tagsPattern": "pattern", - }, - }, + {{Plugin: "refdiff", Options: map[string]interface{}{"tagsLimit": 10, "tagsOrder": "reverse semver", "tagsPattern": "pattern"}}}, + } + assert.Equal(t, expectPlans, actualPlans) +} + +func TestMakeProjectWorkItemStage(t *testing.T) { + const ( + orgName = "test-org" + projName = "test-project" + ) + conn := &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}, + } + board := &models.AzuredevopsBoard{ + Scope: common.Scope{ConnectionId: connectionID}, + AzureDevOpsPK: models.AzureDevOpsPK{ + OrganizationId: orgName, + ProjectId: projName, }, + TeamId: azuredevopsTeamId, + } + sc := &models.AzuredevopsScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, } - assert.Equal(t, expectPlans, actualPlans) + stage := makeProjectWorkItemStage(conn, board, sc) + assert.Equal(t, 1, len(stage)) + task := stage[0] + assert.Equal(t, "azuredevops_go", task.Plugin) + assert.Equal(t, []string{tasks.CollectWorkItemsMeta.Name, tasks.ExtractWorkItemsMeta.Name}, task.Subtasks) + assert.Equal(t, tasks.ScopeKindProject, task.Options["scopeKind"]) + assert.Equal(t, projName, task.Options["name"]) + assert.Equal(t, connectionID, task.Options["connectionId"]) + assert.Equal(t, orgName, task.Options["organizationId"]) + assert.Equal(t, projName, task.Options["projectId"]) +} + +func TestFilterSubtaskMetasByName(t *testing.T) { + metas := []plugin.SubTaskMeta{ + tasks.CollectWorkItemsMeta, + tasks.ExtractWorkItemsMeta, + tasks.CollectApiPullRequestsMeta, + } + filtered := filterSubtaskMetasByName(metas, tasks.CollectWorkItemsMeta.Name, tasks.ExtractWorkItemsMeta.Name) + assert.Equal(t, 1, len(filtered)) + assert.Equal(t, tasks.CollectApiPullRequestsMeta.Name, filtered[0].Name) } -func TestMakeRemoteRepoScopes(t *testing.T) { - mockAzuredevopsPlugin(t) +func TestBoardPlanExcludesWorkItemCollect(t *testing.T) { + subtaskMetas := []plugin.SubTaskMeta{ + tasks.CollectWorkItemsMeta, + tasks.ExtractWorkItemsMeta, + tasks.CollectApiPullRequestsMeta, + tasks.ConvertWorkItemsMeta, + } + boardMetas := filterSubtaskMetasByName(subtaskMetas, + tasks.CollectWorkItemsMeta.Name, tasks.ExtractWorkItemsMeta.Name) + + for _, m := range boardMetas { + assert.NotEqual(t, tasks.CollectWorkItemsMeta.Name, m.Name) + assert.NotEqual(t, tasks.ExtractWorkItemsMeta.Name, m.Name) + } +} +func TestMakeRemoteRepoScopesNoTicketBoard(t *testing.T) { data := []struct { Name string Type string Private bool ExpectedScopes []string }{ - {Name: "Azure DevOps Repository", Type: models.RepositoryTypeADO, Private: false, ExpectedScopes: []string{"*code.Repo", "*ticket.Board", "*devops.CicdScope"}}, - + {Name: "Azure DevOps Repository", Type: models.RepositoryTypeADO, Private: false, ExpectedScopes: []string{"*code.Repo", "*devops.CicdScope"}}, {Name: "Public GitHub Repository", Type: models.RepositoryTypeGithub, Private: false, ExpectedScopes: []string{"*code.Repo", "*devops.CicdScope"}}, - {Name: "Private GitHub Repository", Type: models.RepositoryTypeGithub, Private: true, ExpectedScopes: []string{"*devops.CicdScope"}}, } - for _, d := range data { - t.Run(d.Name, func(t *testing.T) { - id := strings.ToLower(d.Name) - id = strings.ReplaceAll(id, " ", "-") - actualScopes, err := makeScopeV200( - connectionID, - []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ - { - Scope: models.AzuredevopsRepo{ - Scope: common.Scope{ - ConnectionId: connectionID, - }, - Id: id, - Type: d.Type, - Name: d.Name, - IsPrivate: d.Private, - }, - ScopeConfig: &models.AzuredevopsScopeConfig{ - ScopeConfig: common.ScopeConfig{ - Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, - plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW}, - }, - }, - }, + id := strings.ReplaceAll(strings.ToLower(d.Name), " ", "-") + _, actualScopes, err := makeRepoPipelinePlan( + nil, &models.AzuredevopsConnection{BaseConnection: api.BaseConnection{Model: common.Model{ID: connectionID}}}, + &models.AzuredevopsRepo{ + Scope: common.Scope{ConnectionId: connectionID}, Id: id, + Type: d.Type, Name: d.Name, IsPrivate: d.Private, }, + &models.AzuredevopsScopeConfig{ScopeConfig: common.ScopeConfig{ + Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW}, + }}, ) assert.Nil(t, err) - var count int - + count := 0 for _, s := range actualScopes { - xType := reflect.TypeOf(s) - assert.Contains(t, d.ExpectedScopes, xType.String()) + assert.Contains(t, d.ExpectedScopes, reflect.TypeOf(s).String()) count++ } - assert.Equal(t, count, len(d.ExpectedScopes)) + assert.Equal(t, len(d.ExpectedScopes), count) }) - } } diff --git a/backend/plugins/azuredevops_go/api/init.go b/backend/plugins/azuredevops_go/api/init.go index 0401e2b3662..e285cb965d6 100644 --- a/backend/plugins/azuredevops_go/api/init.go +++ b/backend/plugins/azuredevops_go/api/init.go @@ -28,15 +28,31 @@ import ( var vld *validator.Validate var basicRes context.BasicRes -var dsHelper *api.DsHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, models.AzuredevopsScopeConfig] +var dsHelper *api.DsHelper[models.AzuredevopsConnection, models.AzuredevopsBoard, models.AzuredevopsScopeConfig] +var repoDsHelper *api.DsHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, models.AzuredevopsScopeConfig] var raProxy *api.DsRemoteApiProxyHelper[models.AzuredevopsConnection] -var raScopeList *api.DsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, AzuredevopsRemotePagination] -var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.AzuredevopsConnection, models.AzuredevopsRepo] +var raScopeList *api.DsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsBoard, AzuredevopsRemotePagination] +var raRepoScopeList *api.DsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, AzuredevopsRemotePagination] +var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.AzuredevopsConnection, models.AzuredevopsBoard] func Init(br context.BasicRes, p plugin.PluginMeta) { vld = validator.New() basicRes = br dsHelper = api.NewDataSourceHelper[ + models.AzuredevopsConnection, + models.AzuredevopsBoard, + models.AzuredevopsScopeConfig, + ]( + br, + p.Name(), + []string{}, + func(c models.AzuredevopsConnection) models.AzuredevopsConnection { + return c.Sanitize() + }, + nil, + nil, + ) + repoDsHelper = api.NewDataSourceHelper[ models.AzuredevopsConnection, models.AzuredevopsRepo, models.AzuredevopsScopeConfig, @@ -51,7 +67,7 @@ func Init(br context.BasicRes, p plugin.PluginMeta) { nil, ) raProxy = api.NewDsRemoteApiProxyHelper[models.AzuredevopsConnection](dsHelper.ConnApi.ModelApiHelper) - raScopeList = api.NewDsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, AzuredevopsRemotePagination](raProxy, listAzuredevopsRemoteScopes) - raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.AzuredevopsConnection, models.AzuredevopsRepo](raProxy, nil) - + raScopeList = api.NewDsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsBoard, AzuredevopsRemotePagination](raProxy, listAzuredevopsBoardRemoteScopes) + raRepoScopeList = api.NewDsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, AzuredevopsRemotePagination](raProxy, listAzuredevopsRepoRemoteScopes) + raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.AzuredevopsConnection, models.AzuredevopsBoard](raProxy, nil) } diff --git a/backend/plugins/azuredevops_go/api/remote_board_helper.go b/backend/plugins/azuredevops_go/api/remote_board_helper.go new file mode 100644 index 00000000000..c0de66ae4e6 --- /dev/null +++ b/backend/plugins/azuredevops_go/api/remote_board_helper.go @@ -0,0 +1,117 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +const ( + idSeparator = "/" + maxConcurrency = 10 +) + +var supportedSourceRepositories = []string{"github", "githubenterprise", "bitbucket", "git"} + +type AzuredevopsRemotePagination struct { + Skip int + Top int +} + +func listAzuredevopsBoardRemoteScopes( + connection *models.AzuredevopsConnection, + apiClient plugin.ApiClient, + groupId string, + page AzuredevopsRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsBoard], + nextPage *AzuredevopsRemotePagination, + err errors.Error, +) { + org := connection.Organization + vsc := azuredevops.NewClient(connection, apiClient, "https://app.vssps.visualstudio.com") + + if groupId == "" { + return listBoardProjects(vsc, page, org) + } + + id := strings.Split(groupId, idSeparator) + if len(id) != 2 { + return nil, nil, errors.BadInput.New("invalid group id") + } + return listAzuredevopsTeams(vsc, id[0], id[1]) +} + +func listBoardProjects(vsc azuredevops.Client, _ AzuredevopsRemotePagination, org string) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsBoard], + nextPage *AzuredevopsRemotePagination, + err errors.Error, +) { + projects, err := listProjectsForOrg(vsc, org) + if err != nil { + return nil, nil, err + } + for _, p := range projects { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsBoard]{ + Id: p.OrgId + idSeparator + p.Name, + Type: api.RAS_ENTRY_TYPE_GROUP, + Name: p.Name, + }) + } + return children, nil, nil +} + +func listAzuredevopsTeams( + vsc azuredevops.Client, + orgId, projectId string, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsBoard], + nextPage *AzuredevopsRemotePagination, + err errors.Error, +) { + teams, err := vsc.GetTeams(azuredevops.GetTeamsArgs{OrgId: orgId, ProjectId: projectId}) + if err != nil { + return nil, nil, err + } + pID := orgId + idSeparator + projectId + for _, t := range teams { + board := models.AzuredevopsBoard{ + TeamId: t.Id, + Name: t.Name, + Url: t.Url, + } + board.ProjectId = projectId + board.OrganizationId = orgId + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsBoard]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &pID, + Id: t.Id, + Name: t.Name, + FullName: projectId + "/" + t.Name, + Data: &board, + }) + } + return children, nil, nil +} diff --git a/backend/plugins/azuredevops_go/api/remote_helper.go b/backend/plugins/azuredevops_go/api/remote_repo_helper.go similarity index 71% rename from backend/plugins/azuredevops_go/api/remote_helper.go rename to backend/plugins/azuredevops_go/api/remote_repo_helper.go index 00110daf673..0b446aa7f36 100644 --- a/backend/plugins/azuredevops_go/api/remote_helper.go +++ b/backend/plugins/azuredevops_go/api/remote_repo_helper.go @@ -20,6 +20,10 @@ package api import ( "context" "fmt" + "strconv" + "strings" + "sync" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -28,26 +32,14 @@ import ( "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" "golang.org/x/exp/slices" "golang.org/x/sync/errgroup" - "strconv" - "strings" - "sync" ) -const ( - idSeparator = "/" - maxConcurrency = 10 -) - -type AzuredevopsRemotePagination struct { - Skip int - Top int +type projectRef struct { + OrgId string + Name string } -// https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/?view=azure-devops -// https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/multi-repo-checkout?view=azure-devops -var supportedSourceRepositories = []string{"github", "githubenterprise", "bitbucket", "git"} - -func listAzuredevopsRemoteScopes( +func listAzuredevopsRepoRemoteScopes( connection *models.AzuredevopsConnection, apiClient plugin.ApiClient, groupId string, @@ -57,40 +49,56 @@ func listAzuredevopsRemoteScopes( nextPage *AzuredevopsRemotePagination, err errors.Error, ) { - org := connection.Organization vsc := azuredevops.NewClient(connection, apiClient, "https://app.vssps.visualstudio.com") if groupId == "" { - return listAzuredevopsProjects(vsc, page, org) + return listRepoProjects(vsc, page, org) } id := strings.Split(groupId, idSeparator) + if len(id) != 2 { + return nil, nil, errors.BadInput.New("invalid group id") + } if remote, err := listRemoteRepos(vsc, id[0], id[1]); err == nil { children = append(children, remote...) } - - if remote, err := listAzuredevopsRepos(vsc, id[0], id[1]); err == nil { - children = append(children, remote...) + if repos, err := listAzuredevopsRepos(vsc, id[0], id[1]); err == nil { + children = append(children, repos...) } return children, nextPage, nil } -func listAzuredevopsProjects(vsc azuredevops.Client, _ AzuredevopsRemotePagination, org string) ( +func listRepoProjects(vsc azuredevops.Client, _ AzuredevopsRemotePagination, org string) ( children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo], nextPage *AzuredevopsRemotePagination, - err errors.Error) { + err errors.Error, +) { + projects, err := listProjectsForOrg(vsc, org) + if err != nil { + return nil, nil, err + } + for _, p := range projects { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{ + Id: p.OrgId + idSeparator + p.Name, + Type: api.RAS_ENTRY_TYPE_GROUP, + Name: p.Name, + }) + } + return children, nil, nil +} +func listProjectsForOrg(vsc azuredevops.Client, org string) ([]projectRef, errors.Error) { var accounts azuredevops.AccountResponse if org == "" { profile, err := vsc.GetUserProfile() if err != nil { - return nil, nil, err + return nil, err } accounts, err = vsc.GetUserAccounts(profile.Id) if err != nil { - return nil, nil, err + return nil, err } } else { accounts = append(accounts, azuredevops.Account{AccountName: org}) @@ -98,37 +106,31 @@ func listAzuredevopsProjects(vsc azuredevops.Client, _ AzuredevopsRemotePaginati g, _ := errgroup.WithContext(context.Background()) g.SetLimit(maxConcurrency) - var mu sync.Mutex + var projects []projectRef for _, v := range accounts { accountName := v.AccountName g.Go(func() error { - args := azuredevops.GetProjectsArgs{ - OrgId: accountName, - } - projects, err := vsc.GetProjects(args) + args := azuredevops.GetProjectsArgs{OrgId: accountName} + projs, err := vsc.GetProjects(args) if err != nil { return err } - - var tmp []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo] - for _, vv := range projects { - tmp = append(tmp, dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{ - Id: accountName + idSeparator + vv.Name, - Type: api.RAS_ENTRY_TYPE_GROUP, - Name: vv.Name, - }) + var tmp []projectRef + for _, vv := range projs { + tmp = append(tmp, projectRef{OrgId: accountName, Name: vv.Name}) } mu.Lock() - children = append(children, tmp...) + projects = append(projects, tmp...) mu.Unlock() return nil }) } - - err = errors.Convert(g.Wait()) - return + if err := g.Wait(); err != nil { + return nil, errors.Convert(err) + } + return projects, nil } func listAzuredevopsRepos( @@ -136,23 +138,17 @@ func listAzuredevopsRepos( orgId, projectId string, ) ( children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo], - err errors.Error) { - - args := azuredevops.GetRepositoriesArgs{ - OrgId: orgId, - ProjectId: projectId, - } - + err errors.Error, +) { + args := azuredevops.GetRepositoriesArgs{OrgId: orgId, ProjectId: projectId} repos, err := vsc.GetRepositories(args) if err != nil { return nil, err } - for _, v := range repos { if v.IsDisabled { continue } - pID := orgId + idSeparator + projectId repo := models.AzuredevopsRepo{ Id: v.Id, @@ -160,7 +156,6 @@ func listAzuredevopsRepos( Name: v.Name, Url: v.Url, RemoteUrl: v.RemoteUrl, - IsFork: false, } repo.ProjectId = projectId repo.OrganizationId = orgId @@ -181,36 +176,27 @@ func listRemoteRepos( orgId, projectId string, ) ( children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo], - err errors.Error) { - - args := azuredevops.GetServiceEndpointsArgs{ - OrgId: orgId, - ProjectId: projectId, - } - + err errors.Error, +) { + args := azuredevops.GetServiceEndpointsArgs{OrgId: orgId, ProjectId: projectId} endpoints, err := vsc.GetServiceEndpoints(args) if err != nil { return nil, err } - var mu sync.Mutex var remoteRepos []azuredevops.RemoteRepository - g, _ := errgroup.WithContext(context.Background()) g.SetLimit(maxConcurrency) - for _, v := range endpoints { if !slices.Contains(supportedSourceRepositories, v.Type) { continue } - remoteRepoArgs := azuredevops.GetRemoteRepositoriesArgs{ ProjectId: projectId, OrgId: orgId, Provider: v.Type, ServiceEndpoint: v.Id, } - g.Go(func() error { repos, err := vsc.GetRemoteRepositories(remoteRepoArgs) mu.Lock() @@ -219,22 +205,14 @@ func listRemoteRepos( return err }) } - if err := g.Wait(); err != nil { - return nil, errors.Internal.Wrap(err, "failed to call 'GetRemoteRepositories', falling back to empty list") + return nil, errors.Internal.Wrap(err, "failed to call GetRemoteRepositories") } - for _, v := range remoteRepos { pID := orgId + idSeparator + projectId isFork, _ := strconv.ParseBool(v.Properties.IsFork) isPrivate, _ := strconv.ParseBool(v.Properties.IsPrivate) - - // IDs must not contain URL reserved characters (e.g., "/"), as this breaks the routing in the scope API. - // Accessing /plugins/azuredevops_go/connections//apache/incubator-devlake results in a 404 error, where - // "apache/incubator-devlake" is the repository ID returned by ADOs sourceProviders API. - // Therefore, we are creating our own ID, by combining the Service Connection and the External ID remoteId := fmt.Sprintf("%s-%s", v.Properties.ConnectedServiceId, v.Properties.ExternalId) - repo := models.AzuredevopsRepo{ Id: remoteId, Type: v.SourceProviderName, @@ -245,7 +223,6 @@ func listRemoteRepos( IsFork: isFork, IsPrivate: isPrivate, } - repo.ProjectId = projectId repo.OrganizationId = orgId children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{ diff --git a/backend/plugins/azuredevops_go/api/repo_api.go b/backend/plugins/azuredevops_go/api/repo_api.go new file mode 100644 index 00000000000..3c27ddfea50 --- /dev/null +++ b/backend/plugins/azuredevops_go/api/repo_api.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +type PutReposReqBody api.PutScopesReqBody[models.AzuredevopsRepo] +type RepoScopeDetail api.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig] + +func PutRepos(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return repoDsHelper.ScopeApi.PutMultiple(input) +} + +func PatchRepo(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return repoDsHelper.ScopeApi.Patch(input) +} + +func GetRepos(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return repoDsHelper.ScopeApi.GetPage(input) +} + +func GetRepo(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return repoDsHelper.ScopeApi.GetScopeDetail(input) +} + +func DeleteRepo(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return repoDsHelper.ScopeApi.Delete(input) +} + +func RemoteRepos(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raRepoScopeList.Get(input) +} + +func SearchRemoteRepos(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return nil, errors.Default.New("search remote repos is not implemented") +} diff --git a/backend/plugins/azuredevops_go/api/scope_api.go b/backend/plugins/azuredevops_go/api/scope_api.go index f7e168880f5..611c31ea381 100644 --- a/backend/plugins/azuredevops_go/api/scope_api.go +++ b/backend/plugins/azuredevops_go/api/scope_api.go @@ -24,84 +24,25 @@ import ( "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" ) -type PutScopesReqBody api.PutScopesReqBody[models.AzuredevopsRepo] -type ScopeDetail api.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig] +type PutScopesReqBody api.PutScopesReqBody[models.AzuredevopsBoard] +type ScopeDetail api.ScopeDetail[models.AzuredevopsBoard, models.AzuredevopsScopeConfig] -// PutScopes create or update Azure DevOps repo -// @Summary create or update Azure DevOps repo -// @Description Create or update Azure DevOps repo -// @Tags plugins/azuredevops -// @Accept application/json -// @Param connectionId path int true "connection ID" -// @Param scope body PutScopesReqBody true "json" -// @Success 200 {object} []models.AzuredevopsRepo -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/azuredevops/connections/{connectionId}/scopes [PUT] func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.PutMultiple(input) } -// PatchScope patch to Azure DevOps repo -// @Summary patch to Azure DevOps repo -// @Description patch to Azure DevOps repo -// @Tags plugins/azuredevops -// @Accept application/json -// @Param connectionId path int true "connection ID" -// @Param scopeId path int true "scope ID" -// @Param scope body models.AzuredevopsRepo true "json" -// @Success 200 {object} models.AzuredevopsRepo -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/azuredevops/connections/{connectionId}/scopes/{scopeId} [PATCH] func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.Patch(input) } -// GetScopes get Azure DevOps repos -// @Summary get Azure DevOps repos -// @Description get Azure DevOps repos -// @Tags plugins/azuredevops -// @Param connectionId path int true "connection ID" -// @Param searchTerm query string false "search term for scope name" -// @Param pageSize query int false "page size, default 50" -// @Param page query int false "page size, default 1" -// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" -// @Success 200 {object} []ScopeDetail -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/azuredevops/connections/{connectionId}/scopes [GET] func GetScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.GetPage(input) } -// GetScope get one Azure DevOps repo -// @Summary get one Azure DevOps repo -// @Description get one Azure DevOps repo -// @Tags plugins/azuredevops -// @Param connectionId path int true "connection ID" -// @Param scopeId path int true "scope ID" -// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" -// @Success 200 {object} ScopeDetail -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/azuredevops/connections/{connectionId}/scopes/{scopeId} [GET] func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.GetScopeDetail(input) } -// DeleteScope delete plugin data associated with the scope and optionally the scope itself -// @Summary delete plugin data associated with the scope and optionally the scope itself -// @Description delete data associated with plugin scope -// @Tags plugins/azuredevops -// @Param connectionId path int true "connection ID" -// @Param scopeId path int true "scope ID" -// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" -// @Success 200 {object} models.AzuredevopsRepo -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/azuredevops/connections/{connectionId}/scopes/{scopeId} [DELETE] func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.Delete(input) } diff --git a/backend/plugins/azuredevops_go/api/scope_config_api.go b/backend/plugins/azuredevops_go/api/scope_config_api.go index 6294cb5c7da..bbb34867a2a 100644 --- a/backend/plugins/azuredevops_go/api/scope_config_api.go +++ b/backend/plugins/azuredevops_go/api/scope_config_api.go @@ -92,10 +92,19 @@ func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutp // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/azuredevops/scope-config/{scopeConfigId}/projects [GET] -func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { +func GetBoardsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) } +func GetReposByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return repoDsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} + +// GetProjectsByScopeConfig is deprecated; use GetBoardsByScopeConfig. +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return GetBoardsByScopeConfig(input) +} + // DeleteScopeConfig delete a scope config // @Summary delete a scope config // @Description delete a scope config diff --git a/backend/plugins/azuredevops_go/e2e/board_test.go b/backend/plugins/azuredevops_go/e2e/board_test.go new file mode 100644 index 00000000000..d74fcfca79e --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/board_test.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/tasks" +) + +func TestAzureDevopsBoardDataFlow(t *testing.T) { + var azuredevops impl.Azuredevops + dataflowTester := e2ehelper.NewDataFlowTester(t, "azuredevops_go", azuredevops) + + taskData := &tasks.AzuredevopsTaskData{ + Options: &tasks.AzuredevopsOptions{ + ConnectionId: 1, + OrganizationId: "devlake", + ProjectId: "project-1", + TeamId: "team-1", + ScopeConfig: new(models.AzuredevopsScopeConfig), + }, + } + + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_boards.csv", &models.AzuredevopsBoard{}) + dataflowTester.Subtask(tasks.ConvertBoardMeta, taskData) + dataflowTester.VerifyTable( + ticket.Board{}, + "./snapshot_tables/boards.csv", + []string{"id", "name", "url", "type"}, + ) +} diff --git a/backend/plugins/azuredevops_go/e2e/iteration_test.go b/backend/plugins/azuredevops_go/e2e/iteration_test.go new file mode 100644 index 00000000000..8c31b404211 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/iteration_test.go @@ -0,0 +1,68 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/tasks" +) + +func TestAzureDevopsIterationDataFlow(t *testing.T) { + var azuredevops impl.Azuredevops + dataflowTester := e2ehelper.NewDataFlowTester(t, "azuredevops_go", azuredevops) + + taskData := &tasks.AzuredevopsTaskData{ + Options: &tasks.AzuredevopsOptions{ + ConnectionId: 1, + OrganizationId: "devlake", + ProjectId: "project-1", + TeamId: "team-1", + ScopeConfig: new(models.AzuredevopsScopeConfig), + }, + } + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_iterations.csv", &models.AzuredevopsIteration{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_board_iterations.csv", &models.AzuredevopsBoardIteration{}) + dataflowTester.FlushTabler(&ticket.Sprint{}) + dataflowTester.FlushTabler(&ticket.BoardSprint{}) + dataflowTester.Subtask(tasks.ConvertIterationsMeta, taskData) + dataflowTester.VerifyTable( + ticket.Sprint{}, + "./snapshot_tables/sprints.csv", + []string{"id", "name", "started_date", "ended_date"}, + ) + dataflowTester.VerifyTable( + ticket.BoardSprint{}, + "./snapshot_tables/board_sprints.csv", + []string{"board_id", "sprint_id"}, + ) + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_iteration_work_items.csv", &models.AzuredevopsIterationWorkItem{}) + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertIterationWorkItemsMeta, taskData) + dataflowTester.VerifyTable( + ticket.SprintIssue{}, + "./snapshot_tables/sprint_issues.csv", + []string{"sprint_id", "issue_id"}, + ) +} diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_iterations.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_iterations.csv new file mode 100644 index 00000000000..28be0f9ab94 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_iterations.csv @@ -0,0 +1,2 @@ +connection_id,team_id,iteration_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,team-1,iter-1,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_work_items.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_work_items.csv new file mode 100644 index 00000000000..0b9b81d6632 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_board_work_items.csv @@ -0,0 +1,2 @@ +connection_id,team_id,work_item_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,team-1,100,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_boards.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_boards.csv new file mode 100644 index 00000000000..5e67a993653 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_boards.csv @@ -0,0 +1,2 @@ +connection_id,team_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,organization_id,project_id,name,url,scope_config_id +1,team-1,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,,devlake,project-1,Team Alpha,https://dev.azure.com/devlake/project-1/_settings/teams/team-1,0 diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iteration_work_items.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iteration_work_items.csv new file mode 100644 index 00000000000..c28a462a9c9 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iteration_work_items.csv @@ -0,0 +1,2 @@ +connection_id,iteration_id,work_item_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,iter-1,100,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iterations.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iterations.csv new file mode 100644 index 00000000000..830d27be893 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_iterations.csv @@ -0,0 +1,2 @@ +connection_id,iteration_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,team_id,name,path,start_date,finish_date +1,iter-1,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,,team-1,Sprint 1,project-1\Sprint 1,2024-01-01 00:00:00,2024-01-14 23:59:59 diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelog_items.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelog_items.csv new file mode 100644 index 00000000000..67113d45054 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelog_items.csv @@ -0,0 +1,2 @@ +connection_id,changelog_id,field,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,field_type,from_value,from_string,to_value,to_string +1,1,Sprint,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,,,,project-1\Sprint 1,,project-1\Sprint 2 diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelogs.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelogs.csv new file mode 100644 index 00000000000..97ca317fc86 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_item_changelogs.csv @@ -0,0 +1,2 @@ +connection_id,changelog_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,work_item_id,author_id,author_name,created,work_item_updated +1,1,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,,100,author-1,Alice,2024-01-10 12:00:00, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_items.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_items.csv new file mode 100644 index 00000000000..f6cfad79567 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/_tool_azuredevops_go_work_items.csv @@ -0,0 +1,2 @@ +connection_id,work_item_id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,team_id,project_id,issue_key,title,type,state,priority,story_point,epic_key,iteration_path,creator_id,creator_name,assignee_id,assignee_name,created_date,updated_date,resolution_date,lead_time_minutes,std_type,std_status,changelog_total +1,100,2024-02-04 12:00:00.847,2024-02-04 12:00:00.847,,,0,,team-1,project-1,100,User story one,User Story,Done,2,3,,,creator-1,Alice,assignee-1,Bob,2024-01-01 10:00:00,2024-01-15 10:00:00,2024-01-15 10:00:00,20160,REQUIREMENT,DONE,0 diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/board_issues.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..551e719a393 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,2 @@ +board_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +azuredevops_go:AzuredevopsBoard:1:team-1,azuredevops_go:AzuredevopsWorkItem:1:100,,,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/board_sprints.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/board_sprints.csv new file mode 100644 index 00000000000..09942f6f0c8 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/board_sprints.csv @@ -0,0 +1,2 @@ +board_id,sprint_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +azuredevops_go:AzuredevopsBoard:1:team-1,azuredevops_go:AzuredevopsIteration:1:iter-1,,,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/boards.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..9044f3427da --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +azuredevops_go:AzuredevopsBoard:1:team-1,project-1/Team Alpha,,https://dev.azure.com/devlake/project-1/_settings/teams/team-1,,team,,,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/issue_changelogs.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/issue_changelogs.csv new file mode 100644 index 00000000000..a05314645ab --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/issue_changelogs.csv @@ -0,0 +1,2 @@ +id,issue_id,field_name,original_from_value,original_to_value,from_value,to_value,created_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +azuredevops_go:AzuredevopsWorkItemChangelogItem:1:1:Sprint,azuredevops_go:AzuredevopsWorkItem:1:100,Sprint,,project-1\Sprint 1,,project-1\Sprint 2,2024-01-10 12:00:00,,,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/issues.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..a669c3e6756 --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/issues.csv @@ -0,0 +1,2 @@ +id,issue_key,title,description,type,status,story_point,epic_key,original_type,original_status,created_date,updated_date,resolution_date,lead_time_minutes,priority,assignee_id,assignee_name,parent_issue_id,severity,component,url,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +azuredevops_go:AzuredevopsWorkItem:1:100,100,User story one,,REQUIREMENT,DONE,3,,User Story,Done,2024-01-01 10:00:00,2024-01-15 10:00:00,2024-01-15 10:00:00,20160,2,azuredevops_go:AzuredevopsUser:1:assignee-1,Bob,,,,,,,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/sprint_issues.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/sprint_issues.csv new file mode 100644 index 00000000000..8fbb5636cca --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/sprint_issues.csv @@ -0,0 +1,2 @@ +sprint_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +azuredevops_go:AzuredevopsIteration:1:iter-1,azuredevops_go:AzuredevopsWorkItem:1:100,,,, diff --git a/backend/plugins/azuredevops_go/e2e/snapshot_tables/sprints.csv b/backend/plugins/azuredevops_go/e2e/snapshot_tables/sprints.csv new file mode 100644 index 00000000000..63939706d4e --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/snapshot_tables/sprints.csv @@ -0,0 +1,2 @@ +id,name,started_date,ended_date,url,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +azuredevops_go:AzuredevopsIteration:1:iter-1,Sprint 1,2024-01-01 00:00:00,2024-01-14 23:59:59,,,,, diff --git a/backend/plugins/azuredevops_go/e2e/work_item_changelog_test.go b/backend/plugins/azuredevops_go/e2e/work_item_changelog_test.go new file mode 100644 index 00000000000..e324021ceed --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/work_item_changelog_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/tasks" +) + +func TestAzureDevopsWorkItemChangelogDataFlow(t *testing.T) { + var azuredevops impl.Azuredevops + dataflowTester := e2ehelper.NewDataFlowTester(t, "azuredevops_go", azuredevops) + + taskData := &tasks.AzuredevopsTaskData{ + Options: &tasks.AzuredevopsOptions{ + ConnectionId: 1, + OrganizationId: "devlake", + ProjectId: "project-1", + TeamId: "team-1", + ScopeConfig: new(models.AzuredevopsScopeConfig), + }, + } + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_work_item_changelogs.csv", &models.AzuredevopsWorkItemChangelog{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_work_item_changelog_items.csv", &models.AzuredevopsWorkItemChangelogItem{}) + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertWorkItemChangelogsMeta, taskData) + dataflowTester.VerifyTable( + ticket.IssueChangelogs{}, + "./snapshot_tables/issue_changelogs.csv", + []string{"id", "issue_id", "field_name", "original_from_value", "original_to_value", "created_date"}, + ) +} diff --git a/backend/plugins/azuredevops_go/e2e/work_item_test.go b/backend/plugins/azuredevops_go/e2e/work_item_test.go new file mode 100644 index 00000000000..9473e90c97d --- /dev/null +++ b/backend/plugins/azuredevops_go/e2e/work_item_test.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/tasks" +) + +func TestAzureDevopsWorkItemDataFlow(t *testing.T) { + var azuredevops impl.Azuredevops + dataflowTester := e2ehelper.NewDataFlowTester(t, "azuredevops_go", azuredevops) + + taskData := &tasks.AzuredevopsTaskData{ + Options: &tasks.AzuredevopsOptions{ + ConnectionId: 1, + OrganizationId: "devlake", + ProjectId: "project-1", + TeamId: "team-1", + ScopeConfig: new(models.AzuredevopsScopeConfig), + }, + } + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_work_items.csv", &models.AzuredevopsWorkItem{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_azuredevops_go_board_work_items.csv", &models.AzuredevopsBoardWorkItem{}) + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertWorkItemsMeta, taskData) + dataflowTester.VerifyTable( + ticket.Issue{}, + "./snapshot_tables/issues.csv", + []string{"id", "issue_key", "title", "type", "status", "story_point", "assignee_name"}, + ) + dataflowTester.VerifyTable( + ticket.BoardIssue{}, + "./snapshot_tables/board_issues.csv", + []string{"board_id", "issue_id"}, + ) +} diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go index 44cc01e5238..ceac9fdef8a 100644 --- a/backend/plugins/azuredevops_go/impl/impl.go +++ b/backend/plugins/azuredevops_go/impl/impl.go @@ -66,7 +66,7 @@ func (p Azuredevops) Connection() dal.Tabler { } func (p Azuredevops) Scope() plugin.ToolLayerScope { - return &models.AzuredevopsRepo{} + return &models.AzuredevopsBoard{} } func (p Azuredevops) ScopeConfig() dal.Tabler { @@ -82,17 +82,24 @@ func (p Azuredevops) Init(basicRes context.BasicRes) errors.Error { func (p Azuredevops) GetTablesInfo() []dal.Tabler { return []dal.Tabler{ &models.AzuredevopsBuild{}, + &models.AzuredevopsBoard{}, + &models.AzuredevopsBoardIteration{}, + &models.AzuredevopsBoardWorkItem{}, &models.AzuredevopsCommit{}, &models.AzuredevopsConnection{}, + &models.AzuredevopsIteration{}, + &models.AzuredevopsIterationWorkItem{}, &models.AzuredevopsPrCommit{}, &models.AzuredevopsPrLabel{}, - &models.AzuredevopsProject{}, &models.AzuredevopsPullRequest{}, &models.AzuredevopsRepo{}, &models.AzuredevopsRepoCommit{}, &models.AzuredevopsScopeConfig{}, &models.AzuredevopsTimelineRecord{}, &models.AzuredevopsUser{}, + &models.AzuredevopsWorkItem{}, + &models.AzuredevopsWorkItemChangelog{}, + &models.AzuredevopsWorkItemChangelogItem{}, } } @@ -129,7 +136,11 @@ func (p Azuredevops) PrepareTaskData(taskCtx plugin.TaskContext, options map[str apiClient, err := tasks.CreateApiClient(taskCtx, connection) if err != nil { - return nil, errors.Default.Wrap(err, "failed to retrieve an Azure DevOps connection from the database using the provided connection ID") + return nil, errors.Default.Wrap(err, "failed to create Azure DevOps API client") + } + syncApiClient, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to create sync Azure DevOps API client") } if op.RepositoryId != "" { @@ -140,10 +151,27 @@ func (p Azuredevops) PrepareTaskData(taskCtx plugin.TaskContext, options map[str if op.ScopeConfigId == 0 && scope.ScopeConfigId != 0 { op.ScopeConfigId = scope.ScopeConfigId } + } else if !db.IsErrorNotFound(err) { + return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repository: %s/%s", op.ProjectId, op.RepositoryId)) } + } - if err != nil { - return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find repositors: %s/%s", op.ProjectId, op.RepositoryId)) + if op.TeamId != "" { + var board *models.AzuredevopsBoard + db := taskCtx.GetDal() + err = db.First(&board, dal.Where("connection_id = ? AND team_id = ?", op.ConnectionId, op.TeamId)) + if err == nil { + if op.ScopeConfigId == 0 && board.ScopeConfigId != 0 { + op.ScopeConfigId = board.ScopeConfigId + } + if op.OrganizationId == "" { + op.OrganizationId = board.OrganizationId + } + if op.ProjectId == "" { + op.ProjectId = board.ProjectId + } + } else if !db.IsErrorNotFound(err) { + return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find board team: %s", op.TeamId)) } } @@ -177,6 +205,8 @@ func (p Azuredevops) PrepareTaskData(taskCtx plugin.TaskContext, options map[str taskData := &tasks.AzuredevopsTaskData{ Options: op, ApiClient: apiClient, + SyncApiClient: syncApiClient, + Connection: connection, RegexEnricher: regexEnricher, } @@ -217,6 +247,21 @@ func (p Azuredevops) ApiResources() map[string]map[string]plugin.ApiResourceHand "connections/:connectionId/test": { "POST": api.TestExistingConnection, }, + "connections/:connectionId/repos/:repoId": { + "GET": api.GetRepo, + "PATCH": api.PatchRepo, + "DELETE": api.DeleteRepo, + }, + "connections/:connectionId/repos": { + "GET": api.GetRepos, + "PUT": api.PutRepos, + }, + "connections/:connectionId/remote-repos": { + "GET": api.RemoteRepos, + }, + "connections/:connectionId/search-remote-repos": { + "GET": api.SearchRemoteRepos, + }, "connections/:connectionId/scopes/:scopeId": { "GET": api.GetScope, "PATCH": api.PatchScope, @@ -244,8 +289,11 @@ func (p Azuredevops) ApiResources() map[string]map[string]plugin.ApiResourceHand "connections/:connectionId/proxy/rest/*path": { "GET": api.Proxy, }, - "scope-config/:scopeConfigId/projects": { - "GET": api.GetProjectsByScopeConfig, + "scope-config/:scopeConfigId/repos": { + "GET": api.GetReposByScopeConfig, + }, + "scope-config/:scopeConfigId/boards": { + "GET": api.GetBoardsByScopeConfig, }, } } diff --git a/backend/plugins/azuredevops_go/models/board.go b/backend/plugins/azuredevops_go/models/board.go new file mode 100644 index 00000000000..41854aecfb5 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/board.go @@ -0,0 +1,69 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*AzuredevopsBoard)(nil) + +type AzuredevopsBoard struct { + common.Scope `mapstructure:",squash"` + AzureDevOpsPK `mapstructure:",squash"` + + TeamId string `json:"teamId" mapstructure:"teamId" validate:"required" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Url string `json:"url" mapstructure:"url" gorm:"type:varchar(512)"` +} + +func (b AzuredevopsBoard) ScopeId() string { + return b.TeamId +} + +func (b AzuredevopsBoard) ScopeName() string { + return b.Name +} + +func (b AzuredevopsBoard) ScopeFullName() string { + if b.ProjectId != "" { + return b.ProjectId + "/" + b.Name + } + return b.Name +} + +func (b AzuredevopsBoard) ScopeParams() interface{} { + return &AzuredevopsBoardParams{ + ConnectionId: b.ConnectionId, + OrganizationId: b.OrganizationId, + ProjectId: b.ProjectId, + TeamId: b.TeamId, + } +} + +func (AzuredevopsBoard) TableName() string { + return "_tool_azuredevops_go_boards" +} + +type AzuredevopsBoardParams struct { + ConnectionId uint64 + OrganizationId string + ProjectId string + TeamId string +} diff --git a/backend/plugins/azuredevops_go/models/board_work_item.go b/backend/plugins/azuredevops_go/models/board_work_item.go new file mode 100644 index 00000000000..90f3a1a9294 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/board_work_item.go @@ -0,0 +1,33 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AzuredevopsBoardWorkItem struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamId string `gorm:"primaryKey;type:varchar(255)"` + WorkItemId int `gorm:"primaryKey"` + common.NoPKModel +} + +func (AzuredevopsBoardWorkItem) TableName() string { + return "_tool_azuredevops_go_board_work_items" +} diff --git a/backend/plugins/azuredevops_go/models/iteration.go b/backend/plugins/azuredevops_go/models/iteration.go new file mode 100644 index 00000000000..98450549b3a --- /dev/null +++ b/backend/plugins/azuredevops_go/models/iteration.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type AzuredevopsIteration struct { + ConnectionId uint64 `gorm:"primaryKey"` + IterationId string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"type:varchar(255);index"` + Name string `gorm:"type:varchar(255)"` + Path string `gorm:"type:varchar(512)"` + StartDate *time.Time + FinishDate *time.Time + common.NoPKModel +} + +type AzuredevopsBoardIteration struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamId string `gorm:"primaryKey;type:varchar(255)"` + IterationId string `gorm:"primaryKey;type:varchar(255)"` + common.NoPKModel +} + +type AzuredevopsIterationWorkItem struct { + ConnectionId uint64 `gorm:"primaryKey"` + IterationId string `gorm:"primaryKey;type:varchar(255)"` + WorkItemId int `gorm:"primaryKey"` + ResolutionDate *time.Time + IssueCreatedDate *time.Time + common.NoPKModel +} + +func (AzuredevopsIteration) TableName() string { + return "_tool_azuredevops_go_iterations" +} + +func (AzuredevopsBoardIteration) TableName() string { + return "_tool_azuredevops_go_board_iterations" +} + +func (AzuredevopsIterationWorkItem) TableName() string { + return "_tool_azuredevops_go_iteration_work_items" +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/20260616_add_board_tables.go b/backend/plugins/azuredevops_go/models/migrationscripts/20260616_add_board_tables.go new file mode 100644 index 00000000000..90914d55874 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/migrationscripts/20260616_add_board_tables.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +type addBoardTables struct{} + +func (u *addBoardTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.AzuredevopsBoard{}, + &models.AzuredevopsWorkItem{}, + &models.AzuredevopsBoardWorkItem{}, + &models.AzuredevopsIteration{}, + &models.AzuredevopsBoardIteration{}, + &models.AzuredevopsIterationWorkItem{}, + &models.AzuredevopsWorkItemChangelog{}, + &models.AzuredevopsWorkItemChangelogItem{}, + ) +} + +func (*addBoardTables) Version() uint64 { + return 20260616000001 +} + +func (*addBoardTables) Name() string { + return "Add Azure DevOps Go board and work item tables" +} + +type extendScopeConfigForTicket struct{} + +func (u *extendScopeConfigForTicket) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.AzuredevopsScopeConfig{}, + ) +} + +func (*extendScopeConfigForTicket) Version() uint64 { + return 20260616000002 +} + +func (*extendScopeConfigForTicket) Name() string { + return "Extend Azure DevOps Go scope config for TICKET domain" +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/register.go b/backend/plugins/azuredevops_go/models/migrationscripts/register.go index 59fa7832fe0..53a92ae933b 100644 --- a/backend/plugins/azuredevops_go/models/migrationscripts/register.go +++ b/backend/plugins/azuredevops_go/models/migrationscripts/register.go @@ -26,5 +26,7 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), new(extendRepoTable), + new(addBoardTables), + new(extendScopeConfigForTicket), } } diff --git a/backend/plugins/azuredevops_go/models/scope_config.go b/backend/plugins/azuredevops_go/models/scope_config.go index 0ea0d0b5031..92bc7748fc0 100644 --- a/backend/plugins/azuredevops_go/models/scope_config.go +++ b/backend/plugins/azuredevops_go/models/scope_config.go @@ -25,12 +25,25 @@ import ( var _ plugin.ToolLayerScopeConfig = (*AzuredevopsScopeConfig)(nil) +type StatusMapping struct { + StandardStatus string `json:"standardStatus"` +} + +type StatusMappings map[string]StatusMapping + +type TypeMapping struct { + StandardType string `json:"standardType"` + StatusMappings StatusMappings `json:"statusMappings"` +} + type AzuredevopsScopeConfig struct { common.ScopeConfig `mapstructure:",squash" json:",inline"` - DeploymentPattern string `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"` - ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern"` - Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"` + DeploymentPattern string `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern"` + ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern"` + Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"` + StoryPointField string `mapstructure:"storyPointField,omitempty" json:"storyPointField" gorm:"type:varchar(255)"` + TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"` } // GetConnectionId implements plugin.ToolLayerScopeConfig. diff --git a/backend/plugins/azuredevops_go/models/work_item.go b/backend/plugins/azuredevops_go/models/work_item.go new file mode 100644 index 00000000000..6a3cac2c76d --- /dev/null +++ b/backend/plugins/azuredevops_go/models/work_item.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type AzuredevopsWorkItem struct { + ConnectionId uint64 `gorm:"primaryKey"` + WorkItemId int `gorm:"primaryKey"` + TeamId string `gorm:"type:varchar(255);index"` + ProjectId string `gorm:"type:varchar(255)"` + IssueKey string `gorm:"type:varchar(255)"` + Title string `gorm:"type:text"` + Type string `gorm:"type:varchar(255)"` + State string `gorm:"type:varchar(255)"` + Priority string `gorm:"type:varchar(255)"` + StoryPoint *float64 + EpicKey string `gorm:"type:varchar(255)"` + IterationPath string `gorm:"type:varchar(512)"` + CreatorId string `gorm:"type:varchar(255)"` + CreatorName string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + CreatedDate time.Time + UpdatedDate time.Time `gorm:"index"` + ResolutionDate *time.Time + LeadTimeMinutes *uint + StdType string `gorm:"type:varchar(255)"` + StdStatus string `gorm:"type:varchar(255)"` + ChangelogTotal int + common.NoPKModel +} + +func (AzuredevopsWorkItem) TableName() string { + return "_tool_azuredevops_go_work_items" +} diff --git a/backend/plugins/azuredevops_go/models/work_item_changelog.go b/backend/plugins/azuredevops_go/models/work_item_changelog.go new file mode 100644 index 00000000000..791ba0d18fb --- /dev/null +++ b/backend/plugins/azuredevops_go/models/work_item_changelog.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type AzuredevopsWorkItemChangelog struct { + ConnectionId uint64 `gorm:"primaryKey"` + ChangelogId int `gorm:"primaryKey"` + WorkItemId int `gorm:"index"` + AuthorId string `gorm:"type:varchar(255)"` + AuthorName string `gorm:"type:varchar(255)"` + Created *time.Time `gorm:"index"` + WorkItemUpdated *time.Time + common.NoPKModel +} + +type AzuredevopsWorkItemChangelogItem struct { + ConnectionId uint64 `gorm:"primaryKey"` + ChangelogId int `gorm:"primaryKey"` + Field string `gorm:"primaryKey;type:varchar(255)"` + FieldType string `gorm:"type:varchar(255)"` + FromValue string `gorm:"type:text"` + FromString string `gorm:"type:text"` + ToValue string `gorm:"type:text"` + ToString string `gorm:"type:text"` + common.NoPKModel +} + +func (AzuredevopsWorkItemChangelog) TableName() string { + return "_tool_azuredevops_go_work_item_changelogs" +} + +func (AzuredevopsWorkItemChangelogItem) TableName() string { + return "_tool_azuredevops_go_work_item_changelog_items" +} diff --git a/backend/plugins/azuredevops_go/tasks/board_converter.go b/backend/plugins/azuredevops_go/tasks/board_converter.go new file mode 100644 index 00000000000..be0818a06d3 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/board_converter.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" + "reflect" +) + +const RAW_BOARD_TABLE = "azuredevops_go_api_boards" + +func init() { + RegisterSubtaskMeta(&ConvertBoardMeta) +} + +var ConvertBoardMeta = plugin.SubTaskMeta{ + Name: "convertBoard", + EntryPoint: ConvertBoard, + EnabledByDefault: true, + Description: "Convert tool layer board into domain layer board", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsBoard{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +func ConvertBoard(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + db := taskCtx.GetDal() + idGen := didgen.NewDomainIdGenerator(&models.AzuredevopsBoard{}) + clauses := []dal.Clause{ + dal.From(&models.AzuredevopsBoard{}), + dal.Where("connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_BOARD_TABLE, + }, + InputRowType: reflect.TypeOf(models.AzuredevopsBoard{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + board := inputRow.(*models.AzuredevopsBoard) + return []interface{}{&ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: idGen.Generate(data.Options.ConnectionId, board.TeamId), + }, + Name: board.ScopeFullName(), + Url: board.Url, + Type: "team", + }}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/board_work_item_extractor.go b/backend/plugins/azuredevops_go/tasks/board_work_item_extractor.go new file mode 100644 index 00000000000..bae30cbf335 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/board_work_item_extractor.go @@ -0,0 +1,104 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&ExtractBoardWorkItemsMeta) +} + +var ExtractBoardWorkItemsMeta = plugin.SubTaskMeta{ + Name: "extractBoardWorkItems", + EntryPoint: ExtractBoardWorkItems, + EnabledByDefault: true, + Description: "Map work items to board teams via WIQL", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{models.AzuredevopsBoardWorkItem{}.TableName()}, +} + +// ExtractBoardWorkItems queries ADO WIQL with team context to determine which work items +// belong to this board, then writes the mapping to board_work_items. It runs in board scope only. +func ExtractBoardWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if !data.Options.IsBoardScope() { + return nil + } + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + vsc := azuredevops.NewClient(data.Connection, data.SyncApiClient, data.Connection.GetEndpoint()) + + // WIQL with team context to get the IDs that belong to this team's board + wiql := azuredevops.BuildWiqlQuery(data.Options.ProjectId, "") + ids, err := vsc.QueryWorkItems(azuredevops.QueryWorkItemsArgs{ + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + TeamId: data.Options.TeamId, + Wiql: wiql, + }) + if err != nil { + return err + } + + if len(ids) == 0 { + // Fallback: derive membership from iteration work items collected for this board + ids, err = collectWorkItemIdsFromIterations(taskCtx, data) + if err != nil { + return err + } + logger.Info("WIQL returned 0 results; using %d IDs from iterations for board %s", len(ids), data.Options.TeamId) + } + + logger.Info("Mapping %d work items to board team %s", len(ids), data.Options.TeamId) + + // Replace all existing mappings for this board + if delErr := db.Delete(&models.AzuredevopsBoardWorkItem{}, + dal.Where("connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId)); delErr != nil { + return delErr + } + if len(ids) == 0 { + return nil + } + + const batchSize = 500 + for i := 0; i < len(ids); i += batchSize { + end := i + batchSize + if end > len(ids) { + end = len(ids) + } + batch := make([]*models.AzuredevopsBoardWorkItem, 0, end-i) + for _, id := range ids[i:end] { + batch = append(batch, &models.AzuredevopsBoardWorkItem{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + WorkItemId: id, + }) + } + if createErr := db.CreateOrUpdate(batch); createErr != nil { + return createErr + } + } + return nil +} diff --git a/backend/plugins/azuredevops_go/tasks/commit_collector.go b/backend/plugins/azuredevops_go/tasks/commit_collector.go index fe27de1eca9..fa6096ce3e8 100644 --- a/backend/plugins/azuredevops_go/tasks/commit_collector.go +++ b/backend/plugins/azuredevops_go/tasks/commit_collector.go @@ -34,12 +34,15 @@ var CollectCommitsMeta = plugin.SubTaskMeta{ EntryPoint: CollectApiCommits, EnabledByDefault: false, Description: "Collect commit data from Azure DevOps API", - DomainTypes: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CROSS}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, ProductTables: []string{RawCommitTable}, } func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawCommitTable) + if skipRepoScopeSubtask(data) { + return nil + } collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, diff --git a/backend/plugins/azuredevops_go/tasks/cross_domain_task.go b/backend/plugins/azuredevops_go/tasks/cross_domain_task.go new file mode 100644 index 00000000000..4af976cfb81 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/cross_domain_task.go @@ -0,0 +1,247 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "regexp" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +var prURLPattern = regexp.MustCompile(`pullrequest/(\d+)`) + +func init() { + RegisterSubtaskMeta(&ConvertPullRequestIssuesMeta) + RegisterSubtaskMeta(&ConvertIssueCommitsMeta) + RegisterSubtaskMeta(&ConvertIssueRepoCommitsMeta) +} + +var ConvertPullRequestIssuesMeta = plugin.SubTaskMeta{ + Name: "convertPullRequestIssues", + EntryPoint: ConvertPullRequestIssues, + EnabledByDefault: true, + Description: "Convert ADO work item to pull request links", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + ProductTables: []string{crossdomain.PullRequestIssue{}.TableName()}, +} + +var ConvertIssueCommitsMeta = plugin.SubTaskMeta{ + Name: "convertIssueCommits", + EntryPoint: ConvertIssueCommits, + EnabledByDefault: true, + Description: "Convert ADO work item artifact commit links", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + ProductTables: []string{crossdomain.IssueCommit{}.TableName()}, +} + +var ConvertIssueRepoCommitsMeta = plugin.SubTaskMeta{ + Name: "convertIssueRepoCommits", + EntryPoint: ConvertIssueRepoCommits, + EnabledByDefault: true, + Description: "Convert ADO issue commits with repo URLs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + ProductTables: []string{crossdomain.IssueRepoCommit{}.TableName()}, + DependencyTables: []string{crossdomain.IssueCommit{}.TableName()}, +} + +type issueRepoCommitRow struct { + IssueId string `gorm:"column:issue_id"` + CommitSha string `gorm:"column:commit_sha"` + RepoUrl string `gorm:"column:repo_url"` +} + +func ConvertPullRequestIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + db := taskCtx.GetDal() + vsc := azuredevops.NewClient(data.Connection, data.SyncApiClient, data.Connection.GetEndpoint()) + issueIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}) + prIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsPullRequest{}) + + // Work items are collected at project level (team_id=""). Query by project_id. + var workItems []models.AzuredevopsWorkItem + if err := db.All(&workItems, dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId)); err != nil { + return err + } + itemsByID, err := getWorkItemsWithRelationsBatch(vsc, data.Options.OrganizationId, data.Options.ProjectId, workItems) + if err != nil { + return err + } + var results []interface{} + for _, wi := range workItems { + item := itemsByID[wi.WorkItemId] + if item == nil { + continue + } + issueId := issueIdGen.Generate(wi.ConnectionId, wi.WorkItemId) + for _, rel := range item.Relations { + if !strings.Contains(rel.Rel, "pullrequest") && !strings.Contains(rel.Url, "pullrequest") { + continue + } + if m := prURLPattern.FindStringSubmatch(rel.Url); len(m) == 2 { + prNum, _ := strconv.Atoi(m[1]) + var pr models.AzuredevopsPullRequest + if db.First(&pr, dal.Where("connection_id = ? AND azuredevops_id = ?", wi.ConnectionId, prNum)) == nil { + results = append(results, &crossdomain.PullRequestIssue{ + PullRequestId: prIdGen.Generate(wi.ConnectionId, pr.AzuredevopsId), + IssueId: issueId, + }) + } + } + } + } + if len(results) == 0 { + return nil + } + return db.Create(results) +} + +func ConvertIssueCommits(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + db := taskCtx.GetDal() + vsc := azuredevops.NewClient(data.Connection, data.SyncApiClient, data.Connection.GetEndpoint()) + issueIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}) + + // Work items are collected at project level (team_id=""). Query by project_id. + var workItems []models.AzuredevopsWorkItem + if err := db.All(&workItems, dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId)); err != nil { + return err + } + itemsByID, err := getWorkItemsWithRelationsBatch(vsc, data.Options.OrganizationId, data.Options.ProjectId, workItems) + if err != nil { + return err + } + var results []interface{} + for _, wi := range workItems { + item := itemsByID[wi.WorkItemId] + if item == nil { + continue + } + issueId := issueIdGen.Generate(wi.ConnectionId, wi.WorkItemId) + for _, rel := range item.Relations { + if !strings.Contains(rel.Rel, "ArtifactLink") { + continue + } + if sha := extractCommitSha(rel.Url); sha != "" { + results = append(results, &crossdomain.IssueCommit{ + IssueId: issueId, + CommitSha: sha, + }) + } + } + } + if len(results) == 0 { + return nil + } + return db.Create(results) +} + +func getWorkItemsWithRelationsBatch( + vsc azuredevops.Client, + orgID string, + projectID string, + workItems []models.AzuredevopsWorkItem, +) (map[int]*azuredevops.WorkItem, errors.Error) { + ids := make([]int, 0, len(workItems)) + for _, wi := range workItems { + if wi.WorkItemId > 0 { + ids = append(ids, wi.WorkItemId) + } + } + items, err := vsc.GetWorkItemsBatch(azuredevops.GetWorkItemsBatchArgs{ + OrgId: orgID, + ProjectId: projectID, + Ids: ids, + Expand: "Relations", + }) + if err != nil { + return nil, err + } + itemsByID := make(map[int]*azuredevops.WorkItem, len(items)) + for i := range items { + itemsByID[items[i].Id] = &items[i] + } + return itemsByID, nil +} + +func extractCommitSha(url string) string { + parts := strings.Split(url, "/") + if len(parts) == 0 { + return "" + } + sha := parts[len(parts)-1] + if len(sha) >= 7 && len(sha) <= 40 { + return sha + } + return "" +} + +func ConvertIssueRepoCommits(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if !data.Options.IsBoardScope() { + return nil + } + db := taskCtx.GetDal() + boardIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsBoard{}) + boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.Select("ic.issue_id, ic.commit_sha, r.url as repo_url"), + dal.From("issue_commits ic"), + dal.Join("inner join commits c on c.sha = ic.commit_sha"), + dal.Join("inner join repo_commits rc on rc.commit_sha = c.sha"), + dal.Join("inner join repos r on r.id = rc.repo_id"), + dal.Join("inner join board_issues bi on bi.issue_id = ic.issue_id"), + dal.Where("bi.board_id = ?", boardId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_WORK_ITEM_TABLE, + }, + InputRowType: reflect.TypeOf(issueRepoCommitRow{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + row := inputRow.(*issueRepoCommitRow) + return []interface{}{&crossdomain.IssueRepoCommit{ + IssueId: row.IssueId, + CommitSha: row.CommitSha, + RepoUrl: row.RepoUrl, + }}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/iteration_converter.go b/backend/plugins/azuredevops_go/tasks/iteration_converter.go new file mode 100644 index 00000000000..6b450210bf4 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/iteration_converter.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +func convertIterationsToDomain(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData) errors.Error { + db := taskCtx.GetDal() + sprintIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsIteration{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsBoard{}) + boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.From(&models.AzuredevopsIteration{}), + dal.Where("connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{Ctx: taskCtx, Options: data.Options, Table: RAW_ITERATION_TABLE}, + InputRowType: reflect.TypeOf(models.AzuredevopsIteration{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + it := inputRow.(*models.AzuredevopsIteration) + sprintId := sprintIdGen.Generate(it.ConnectionId, it.IterationId) + return []interface{}{ + &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: it.Name, + StartedDate: it.StartDate, + EndedDate: it.FinishDate, + }, + &ticket.BoardSprint{BoardId: boardId, SprintId: sprintId}, + }, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +func convertIterationWorkItemsToDomain(taskCtx plugin.SubTaskContext, data *AzuredevopsTaskData) errors.Error { + db := taskCtx.GetDal() + sprintIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsIteration{}) + issueIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}) + + cursor, err := db.Cursor( + dal.From(&models.AzuredevopsIterationWorkItem{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{Ctx: taskCtx, Options: data.Options, Table: RAW_ITERATION_TABLE}, + InputRowType: reflect.TypeOf(models.AzuredevopsIterationWorkItem{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + iw := inputRow.(*models.AzuredevopsIterationWorkItem) + return []interface{}{&ticket.SprintIssue{ + SprintId: sprintIdGen.Generate(iw.ConnectionId, iw.IterationId), + IssueId: issueIdGen.Generate(iw.ConnectionId, iw.WorkItemId), + }}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/iteration_task.go b/backend/plugins/azuredevops_go/tasks/iteration_task.go new file mode 100644 index 00000000000..3aa275af093 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/iteration_task.go @@ -0,0 +1,239 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +const ( + RAW_ITERATION_TABLE = "azuredevops_go_api_iterations" + RAW_ITERATION_WORK_ITEM_TABLE = "azuredevops_go_api_iteration_work_items" +) + +func init() { + RegisterSubtaskMeta(&CollectIterationsMeta) + RegisterSubtaskMeta(&ExtractIterationsMeta) + RegisterSubtaskMeta(&CollectIterationWorkItemsMeta) + RegisterSubtaskMeta(&ExtractIterationWorkItemsMeta) + RegisterSubtaskMeta(&ConvertIterationsMeta) + RegisterSubtaskMeta(&ConvertIterationWorkItemsMeta) +} + +var CollectIterationsMeta = plugin.SubTaskMeta{ + Name: "collectIterations", + EntryPoint: CollectIterations, + EnabledByDefault: true, + Description: "Collect Azure DevOps team iterations", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var ExtractIterationsMeta = plugin.SubTaskMeta{ + Name: "extractIterations", + EntryPoint: ExtractIterations, + EnabledByDefault: true, + Description: "Extract Azure DevOps iterations", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{"_raw_" + RAW_ITERATION_TABLE}, +} + +var CollectIterationWorkItemsMeta = plugin.SubTaskMeta{ + Name: "collectIterationWorkItems", + EntryPoint: CollectIterationWorkItems, + EnabledByDefault: true, + Description: "Collect Azure DevOps iteration work items", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsIteration{}.TableName()}, +} + +var ExtractIterationWorkItemsMeta = plugin.SubTaskMeta{ + Name: "extractIterationWorkItems", + EntryPoint: ExtractIterationWorkItems, + EnabledByDefault: true, + Description: "Extract Azure DevOps iteration work items", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{"_raw_" + RAW_ITERATION_WORK_ITEM_TABLE}, +} + +var ConvertIterationsMeta = plugin.SubTaskMeta{ + Name: "convertIterations", + EntryPoint: ConvertIterations, + EnabledByDefault: true, + Description: "Convert Azure DevOps iterations to domain sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsIteration{}.TableName()}, + ProductTables: []string{"sprints", "board_sprints"}, +} + +var ConvertIterationWorkItemsMeta = plugin.SubTaskMeta{ + Name: "convertIterationWorkItems", + EntryPoint: ConvertIterationWorkItems, + EnabledByDefault: true, + Description: "Convert iteration work item links", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsIterationWorkItem{}.TableName()}, + ProductTables: []string{"sprint_issues"}, +} + +func CollectIterations(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + vsc := azuredevops.NewClient(data.Connection, data.SyncApiClient, data.Connection.GetEndpoint()) + iterations, err := vsc.GetTeamIterations(azuredevops.GetTeamIterationsArgs{ + OrgId: data.Options.OrganizationId, ProjectId: data.Options.ProjectId, TeamId: data.Options.TeamId, + }) + if err != nil { + return err + } + params := data.Options.GetParams() + paramsStr := plugin.MarshalScopeParams(params) + rawTable := "_raw_" + RAW_ITERATION_TABLE + db := taskCtx.GetDal() + _ = db.AutoMigrate(&api.RawData{}, dal.From(rawTable)) + _ = db.Delete(&api.RawData{}, dal.From(rawTable), dal.Where("params = ?", paramsStr)) + rows := make([]*api.RawData, len(iterations)) + for i, it := range iterations { + body, mErr := json.Marshal(it) + if mErr != nil { + return errors.Convert(mErr) + } + rows[i] = &api.RawData{Params: paramsStr, Data: body} + } + if len(rows) > 0 { + return db.Create(rows, dal.From(rawTable)) + } + return nil +} + +func ExtractIterations(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{Ctx: taskCtx, Options: data.Options, Table: RAW_ITERATION_TABLE}, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var it azuredevops.TeamIteration + if err := json.Unmarshal(row.Data, &it); err != nil { + return nil, errors.Convert(err) + } + iteration := &models.AzuredevopsIteration{ + ConnectionId: data.Options.ConnectionId, + IterationId: it.Id, + TeamId: data.Options.TeamId, + Name: it.Name, + Path: it.Path, + StartDate: it.Attributes.StartDate, + FinishDate: it.Attributes.FinishDate, + } + boardIteration := &models.AzuredevopsBoardIteration{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + IterationId: it.Id, + } + return []interface{}{iteration, boardIteration}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ConvertIterations(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + return convertIterationsToDomain(taskCtx, data) +} + +type iterationWorkItemRaw struct { + IterationId string `json:"iterationId"` + WorkItemId int `json:"workItemId"` +} + +func CollectIterationWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + db := taskCtx.GetDal() + vsc := azuredevops.NewClient(data.Connection, data.SyncApiClient, data.Connection.GetEndpoint()) + + var iterations []models.AzuredevopsIteration + if err := db.All(&iterations, dal.Where("connection_id = ? AND team_id = ?", + data.Options.ConnectionId, data.Options.TeamId)); err != nil { + return err + } + + params := data.Options.GetParams() + paramsStr := plugin.MarshalScopeParams(params) + rawTable := "_raw_" + RAW_ITERATION_WORK_ITEM_TABLE + _ = db.AutoMigrate(&api.RawData{}, dal.From(rawTable)) + _ = db.Delete(&api.RawData{}, dal.From(rawTable), dal.Where("params = ?", paramsStr)) + + var rows []*api.RawData + for _, it := range iterations { + ids, err := vsc.GetIterationWorkItems(azuredevops.GetIterationWorkItemsArgs{ + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + TeamId: data.Options.TeamId, + IterationId: it.IterationId, + }) + if err != nil { + taskCtx.GetLogger().Warn(err, "failed to collect iteration work items for %s", it.IterationId) + continue + } + for _, id := range ids { + body, mErr := json.Marshal(iterationWorkItemRaw{IterationId: it.IterationId, WorkItemId: id}) + if mErr != nil { + return errors.Convert(mErr) + } + rows = append(rows, &api.RawData{Params: paramsStr, Data: body}) + } + } + if len(rows) > 0 { + return db.Create(rows, dal.From(rawTable)) + } + return nil +} + +func ExtractIterationWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{Ctx: taskCtx, Options: data.Options, Table: RAW_ITERATION_WORK_ITEM_TABLE}, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var raw iterationWorkItemRaw + if err := json.Unmarshal(row.Data, &raw); err != nil { + return nil, errors.Convert(err) + } + return []interface{}{&models.AzuredevopsIterationWorkItem{ + ConnectionId: data.Options.ConnectionId, + IterationId: raw.IterationId, + WorkItemId: raw.WorkItemId, + }}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ConvertIterationWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + return convertIterationWorkItemsToDomain(taskCtx, data) +} diff --git a/backend/plugins/azuredevops_go/tasks/pr_collector.go b/backend/plugins/azuredevops_go/tasks/pr_collector.go index 30a63dba7bb..4007ac24388 100644 --- a/backend/plugins/azuredevops_go/tasks/pr_collector.go +++ b/backend/plugins/azuredevops_go/tasks/pr_collector.go @@ -38,13 +38,16 @@ var CollectApiPullRequestsMeta = plugin.SubTaskMeta{ EntryPoint: CollectApiPullRequests, EnabledByDefault: true, Description: "Collect PullRequests data from Azure DevOps API, supports timeFilter but not diffSync.", - DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, DependencyTables: []string{}, ProductTables: []string{RawPullRequestTable}, } func CollectApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*AzuredevopsTaskData) + if skipRepoScopeSubtask(data) { + return nil + } rawDataSubTaskArgs := &api.RawDataSubTaskArgs{ Ctx: taskCtx, diff --git a/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go b/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go index dbbddd67b7d..2d321465277 100644 --- a/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go +++ b/backend/plugins/azuredevops_go/tasks/pr_commit_collector.go @@ -39,7 +39,7 @@ var CollectApiPullRequestCommitsMeta = plugin.SubTaskMeta{ EntryPoint: CollectApiPullRequestCommits, EnabledByDefault: true, Description: "Collect PullRequestCommits data from Azure DevOps API.", - DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, DependencyTables: []string{models.AzuredevopsPullRequest{}.TableName()}, ProductTables: []string{RawPrCommitTable}, } @@ -50,6 +50,9 @@ type SimplePr struct { func CollectApiPullRequestCommits(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawPrCommitTable) + if skipRepoScopeSubtask(data) { + return nil + } db := taskCtx.GetDal() cursor, err := db.Cursor( diff --git a/backend/plugins/azuredevops_go/tasks/pr_commit_extractor.go b/backend/plugins/azuredevops_go/tasks/pr_commit_extractor.go index 6e327f31c89..ee64c8720a3 100644 --- a/backend/plugins/azuredevops_go/tasks/pr_commit_extractor.go +++ b/backend/plugins/azuredevops_go/tasks/pr_commit_extractor.go @@ -34,7 +34,7 @@ var ExtractApiPullRequestCommitsMeta = plugin.SubTaskMeta{ EntryPoint: ExtractApiPullRequestCommits, EnabledByDefault: true, Description: "Extract raw pull requests commit data into tool layer table AzuredevopsPullRequestCommit and AzuredevopsRepoCommit", - DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, DependencyTables: []string{RawPrCommitTable}, ProductTables: []string{ models.AzuredevopsPrCommit{}.TableName(), diff --git a/backend/plugins/azuredevops_go/tasks/pr_extractor.go b/backend/plugins/azuredevops_go/tasks/pr_extractor.go index a80b879b253..bab29427a4f 100644 --- a/backend/plugins/azuredevops_go/tasks/pr_extractor.go +++ b/backend/plugins/azuredevops_go/tasks/pr_extractor.go @@ -35,7 +35,7 @@ var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{ EntryPoint: ExtractApiPullRequests, EnabledByDefault: true, Description: "Extract raw PullRequests data into tool layer table azuredevops_pull_requests", - DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, DependencyTables: []string{RawPullRequestTable}, ProductTables: []string{ models.AzuredevopsPullRequest{}.TableName(), diff --git a/backend/plugins/azuredevops_go/tasks/repo_converter.go b/backend/plugins/azuredevops_go/tasks/repo_converter.go index 8baa005e062..1eec543e23f 100644 --- a/backend/plugins/azuredevops_go/tasks/repo_converter.go +++ b/backend/plugins/azuredevops_go/tasks/repo_converter.go @@ -41,10 +41,9 @@ var ConvertRepoMeta = plugin.SubTaskMeta{ Description: "Convert tool layer table _tool_azuredevops_go_repos into domain layer table repos and cicd scope", DomainTypes: []string{ plugin.DOMAIN_TYPE_CODE, - plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW, - plugin.DOMAIN_TYPE_CROSS}, + }, DependencyTables: []string{ models.AzuredevopsRepo{}.TableName(), }, @@ -55,6 +54,9 @@ var ConvertRepoMeta = plugin.SubTaskMeta{ func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, models.AzuredevopsRepo{}.TableName()) + if skipRepoScopeSubtask(data) { + return nil + } db := taskCtx.GetDal() clauses := []dal.Clause{ dal.From(&models.AzuredevopsRepo{}), diff --git a/backend/plugins/azuredevops_go/tasks/shared.go b/backend/plugins/azuredevops_go/tasks/shared.go index e6fe0eca1e2..49eb655ebb5 100644 --- a/backend/plugins/azuredevops_go/tasks/shared.go +++ b/backend/plugins/azuredevops_go/tasks/shared.go @@ -55,6 +55,10 @@ type CustomPageDate struct { ContinuationToken string } +func skipRepoScopeSubtask(data *AzuredevopsTaskData) bool { + return data.Options.IsBoardScope() +} + func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*api.RawDataSubTaskArgs, *AzuredevopsTaskData) { data := taskCtx.GetData().(*AzuredevopsTaskData) RawDataSubTaskArgs := &api.RawDataSubTaskArgs{ diff --git a/backend/plugins/azuredevops_go/tasks/task_data.go b/backend/plugins/azuredevops_go/tasks/task_data.go index 2ac479156ff..d4428fce8c5 100644 --- a/backend/plugins/azuredevops_go/tasks/task_data.go +++ b/backend/plugins/azuredevops_go/tasks/task_data.go @@ -21,11 +21,23 @@ import ( "time" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" ) +const ScopeKindBoard = "board" +const ScopeKindRepo = "repo" +const ScopeKindProject = "project" + +// AzuredevopsProjectParams identifies raw data collected at project level (no team/repo context). +type AzuredevopsProjectParams struct { + ConnectionId uint64 `json:"connectionId"` + OrganizationId string `json:"organizationId"` + ProjectId string `json:"projectId"` +} + type AzuredevopsOptions struct { ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` ProjectId string `json:"projectId" mapstructure:"projectId,omitempty"` @@ -33,6 +45,8 @@ type AzuredevopsOptions struct { RepositoryId string `json:"repositoryId" mapstructure:"repositoryId,omitempty"` RepositoryType string `json:"repositoryType" mapstructure:"repositoryType,omitempty"` ExternalId string `json:"externalId" mapstructure:"externalId,omitempty"` + TeamId string `json:"teamId" mapstructure:"teamId,omitempty"` + ScopeKind string `json:"scopeKind" mapstructure:"scopeKind,omitempty"` ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` @@ -42,6 +56,8 @@ type AzuredevopsOptions struct { type AzuredevopsTaskData struct { Options *AzuredevopsOptions ApiClient *helper.ApiAsyncClient + SyncApiClient plugin.ApiClient + Connection *models.AzuredevopsConnection TimeAfter *time.Time RegexEnricher *helper.RegexEnricher } @@ -52,9 +68,26 @@ func DecodeTaskOptions(options map[string]interface{}) (*AzuredevopsOptions, err if err != nil { return nil, err } + if op.ScopeKind == ScopeKindProject { + // already set by blueprint + } else if op.TeamId != "" { + op.ScopeKind = ScopeKindBoard + } else if op.RepositoryId != "" { + op.ScopeKind = ScopeKindRepo + } return &op, nil } +// GetProjectParams returns the project-level params (connection + org + project, no team/repo). +// Used to scope raw work item data collected once per project. +func (p *AzuredevopsOptions) GetProjectParams() AzuredevopsProjectParams { + return AzuredevopsProjectParams{ + ConnectionId: p.ConnectionId, + OrganizationId: p.OrganizationId, + ProjectId: p.ProjectId, + } +} + type AzuredevopsParams struct { OrganizationId string RepositoryId string @@ -62,9 +95,37 @@ type AzuredevopsParams struct { } func (p *AzuredevopsOptions) GetParams() any { - return AzuredevopsParams{ - OrganizationId: p.OrganizationId, - ProjectId: p.ProjectId, - RepositoryId: p.RepositoryId, + switch p.ScopeKind { + case ScopeKindProject: + return p.GetProjectParams() + case ScopeKindBoard: + return models.AzuredevopsBoardParams{ + ConnectionId: p.ConnectionId, + OrganizationId: p.OrganizationId, + ProjectId: p.ProjectId, + TeamId: p.TeamId, + } + default: + if p.TeamId != "" { + return models.AzuredevopsBoardParams{ + ConnectionId: p.ConnectionId, + OrganizationId: p.OrganizationId, + ProjectId: p.ProjectId, + TeamId: p.TeamId, + } + } + return AzuredevopsParams{ + OrganizationId: p.OrganizationId, + ProjectId: p.ProjectId, + RepositoryId: p.RepositoryId, + } } } + +func (p *AzuredevopsOptions) IsBoardScope() bool { + return p.ScopeKind == ScopeKindBoard || (p.ScopeKind == "" && p.TeamId != "") +} + +func (p *AzuredevopsOptions) IsProjectScope() bool { + return p.ScopeKind == ScopeKindProject +} diff --git a/backend/plugins/azuredevops_go/tasks/type_mapping.go b/backend/plugins/azuredevops_go/tasks/type_mapping.go new file mode 100644 index 00000000000..e54a18473ad --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/type_mapping.go @@ -0,0 +1,114 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +var defaultTypeMappings = map[string]models.TypeMapping{ + "Bug": { + StandardType: ticket.BUG, + StatusMappings: models.StatusMappings{ + "New": {StandardStatus: ticket.TODO}, + "To Do": {StandardStatus: ticket.TODO}, + "Active": {StandardStatus: ticket.IN_PROGRESS}, + "In Progress": {StandardStatus: ticket.IN_PROGRESS}, + "Done": {StandardStatus: ticket.DONE}, + "Closed": {StandardStatus: ticket.DONE}, + "Resolved": {StandardStatus: ticket.DONE}, + }, + }, + "User Story": { + StandardType: ticket.REQUIREMENT, + StatusMappings: models.StatusMappings{ + "New": {StandardStatus: ticket.TODO}, + "To Do": {StandardStatus: ticket.TODO}, + "Active": {StandardStatus: ticket.IN_PROGRESS}, + "In Progress": {StandardStatus: ticket.IN_PROGRESS}, + "Done": {StandardStatus: ticket.DONE}, + "Closed": {StandardStatus: ticket.DONE}, + "Resolved": {StandardStatus: ticket.DONE}, + }, + }, + "Product Backlog Item": { + StandardType: ticket.REQUIREMENT, + }, + "Feature": { + StandardType: ticket.REQUIREMENT, + }, + "Epic": { + StandardType: ticket.REQUIREMENT, + }, + "Task": { + StandardType: ticket.TASK, + }, + "Production Incident": { + StandardType: ticket.INCIDENT, + }, +} + +func mergeTypeMappings(scopeConfig *models.AzuredevopsScopeConfig) map[string]models.TypeMapping { + merged := make(map[string]models.TypeMapping) + for k, v := range defaultTypeMappings { + merged[k] = v + } + if scopeConfig != nil { + for k, v := range scopeConfig.TypeMappings { + merged[k] = v + } + } + return merged +} + +func mapWorkItemTypeAndStatus(wiType, state string, mappings map[string]models.TypeMapping) (stdType, stdStatus string) { + stdType = ticket.SUBTASK + stdStatus = ticket.OTHER + if mapping, ok := mappings[wiType]; ok { + stdType = mapping.StandardType + if statusMapping, ok := mapping.StatusMappings[state]; ok { + stdStatus = statusMapping.StandardStatus + } else { + stdStatus = defaultStatusForState(state) + } + } else { + stdStatus = defaultStatusForState(state) + } + return +} + +func defaultStatusForState(state string) string { + switch state { + case "New", "To Do": + return ticket.TODO + case "Active", "In Progress": + return ticket.IN_PROGRESS + case "Done", "Closed", "Resolved": + return ticket.DONE + default: + return ticket.OTHER + } +} + +func storyPointField(scopeConfig *models.AzuredevopsScopeConfig) string { + if scopeConfig != nil && scopeConfig.StoryPointField != "" { + return scopeConfig.StoryPointField + } + return "Microsoft.VSTS.Scheduling.StoryPoints" +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_changelog_task.go b/backend/plugins/azuredevops_go/tasks/work_item_changelog_task.go new file mode 100644 index 00000000000..246e7f68c17 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_changelog_task.go @@ -0,0 +1,236 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +const RAW_WORK_ITEM_CHANGELOG_TABLE = "azuredevops_go_api_work_item_updates" + +func init() { + RegisterSubtaskMeta(&CollectWorkItemUpdatesMeta) + RegisterSubtaskMeta(&ExtractWorkItemUpdatesMeta) + RegisterSubtaskMeta(&ConvertWorkItemChangelogsMeta) +} + +var CollectWorkItemUpdatesMeta = plugin.SubTaskMeta{ + Name: "collectWorkItemUpdates", + EntryPoint: CollectWorkItemUpdates, + EnabledByDefault: true, + Description: "Collect Azure DevOps work item updates", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.AzuredevopsBoardWorkItem{}.TableName()}, +} + +var ExtractWorkItemUpdatesMeta = plugin.SubTaskMeta{ + Name: "extractWorkItemUpdates", + EntryPoint: ExtractWorkItemUpdates, + EnabledByDefault: true, + Description: "Extract Azure DevOps work item changelogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{"_raw_" + RAW_WORK_ITEM_CHANGELOG_TABLE}, +} + +var ConvertWorkItemChangelogsMeta = plugin.SubTaskMeta{ + Name: "convertWorkItemChangelogs", + EntryPoint: ConvertWorkItemChangelogs, + EnabledByDefault: true, + Description: "Convert work item changelogs to domain layer", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{ticket.IssueChangelogs{}.TableName()}, +} + +func CollectWorkItemUpdates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + db := taskCtx.GetDal() + + // Work items are stored at project level; resolve board membership via board_work_items. + var boardWorkItems []models.AzuredevopsBoardWorkItem + err := db.All(&boardWorkItems, dal.Where( + "connection_id = ? AND team_id = ?", data.Options.ConnectionId, data.Options.TeamId, + )) + if err != nil { + return err + } + vsc := azuredevops.NewClient(data.Connection, data.SyncApiClient, data.Connection.GetEndpoint()) + params := data.Options.GetParams() + paramsStr := plugin.MarshalScopeParams(params) + rawTable := "_raw_" + RAW_WORK_ITEM_CHANGELOG_TABLE + _ = db.AutoMigrate(&api.RawData{}, dal.From(rawTable)) + _ = db.Delete(&api.RawData{}, dal.From(rawTable), dal.Where("params = ?", paramsStr)) + + for _, bwi := range boardWorkItems { + updates, err := vsc.GetWorkItemUpdates(azuredevops.GetWorkItemUpdatesArgs{ + OrgId: data.Options.OrganizationId, WorkItemId: bwi.WorkItemId, + }) + if err != nil { + continue + } + payload := map[string]interface{}{"workItemId": bwi.WorkItemId, "updates": updates} + body, mErr := json.Marshal(payload) + if mErr != nil { + return errors.Convert(mErr) + } + err = db.Create(&api.RawData{Params: paramsStr, Data: body}, dal.From(rawTable)) + if err != nil { + return err + } + } + return nil +} + +func ExtractWorkItemUpdates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{Ctx: taskCtx, Options: data.Options, Table: RAW_WORK_ITEM_CHANGELOG_TABLE}, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var payload struct { + WorkItemId int `json:"workItemId"` + Updates json.RawMessage `json:"updates"` + } + if err := json.Unmarshal(row.Data, &payload); err != nil { + return nil, errors.Convert(err) + } + var updates []map[string]interface{} + if err := json.Unmarshal(payload.Updates, &updates); err != nil { + return nil, errors.Convert(err) + } + var results []interface{} + for _, upd := range updates { + rev := int(upd["rev"].(float64)) + changelog := &models.AzuredevopsWorkItemChangelog{ + ConnectionId: data.Options.ConnectionId, + ChangelogId: rev, + WorkItemId: payload.WorkItemId, + } + if fields, ok := upd["fields"].(map[string]interface{}); ok { + if changed, ok := fields["System.ChangedDate"].(map[string]interface{}); ok { + if newVal, ok := changed["newValue"].(string); ok { + if t, err := parseADOTime(newVal); err == nil { + changelog.Created = &t + } + } + } + } + results = append(results, changelog) + if fields, ok := upd["fields"].(map[string]interface{}); ok { + for field, change := range fields { + if field == "System.ChangedDate" { + continue + } + fieldName := field + if field == "System.IterationPath" { + fieldName = "Sprint" + } + item := &models.AzuredevopsWorkItemChangelogItem{ + ConnectionId: data.Options.ConnectionId, + ChangelogId: rev, + Field: fieldName, + } + if m, ok := change.(map[string]interface{}); ok { + item.FromString = toString(m["oldValue"]) + item.ToString = toString(m["newValue"]) + } + results = append(results, item) + } + } + } + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func ConvertWorkItemChangelogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + db := taskCtx.GetDal() + changelogIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItemChangelogItem{}) + issueIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}) + + cursor, err := db.Cursor( + dal.From(&models.AzuredevopsWorkItemChangelogItem{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{Ctx: taskCtx, Options: data.Options, Table: RAW_WORK_ITEM_CHANGELOG_TABLE}, + InputRowType: reflect.TypeOf(models.AzuredevopsWorkItemChangelogItem{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + item := inputRow.(*models.AzuredevopsWorkItemChangelogItem) + var changelog models.AzuredevopsWorkItemChangelog + _ = db.First(&changelog, dal.Where("connection_id = ? AND changelog_id = ?", + item.ConnectionId, item.ChangelogId)) + createdDate := changelog.Created + if createdDate == nil { + // Fallback to row created_at to avoid zero/invalid datetimes from source payloads. + t := changelog.CreatedAt + if t.IsZero() { + t = time.Now() + } + createdDate = &t + } + return []interface{}{&ticket.IssueChangelogs{ + DomainEntity: domainlayer.DomainEntity{ + Id: changelogIdGen.Generate(item.ConnectionId, item.ChangelogId, item.Field), + }, + IssueId: issueIdGen.Generate(item.ConnectionId, changelog.WorkItemId), + FieldName: item.Field, + OriginalFromValue: item.FromString, + OriginalToValue: item.ToString, + CreatedDate: *createdDate, + }}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +func toString(v interface{}) string { + if v == nil { + return "" + } + if s, ok := v.(string); ok { + return s + } + return strings.TrimSpace(fmt.Sprintf("%v", v)) +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_collector.go b/backend/plugins/azuredevops_go/tasks/work_item_collector.go new file mode 100644 index 00000000000..1c4008e99ca --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_collector.go @@ -0,0 +1,181 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "sort" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" +) + +const RAW_WORK_ITEM_TABLE = "azuredevops_go_api_work_items" +const workItemBatchSize = 200 + +func init() { + RegisterSubtaskMeta(&CollectWorkItemsMeta) +} + +var CollectWorkItemsMeta = plugin.SubTaskMeta{ + Name: "collectWorkItems", + EntryPoint: CollectWorkItems, + EnabledByDefault: true, + Description: "Collect Azure DevOps work items via WIQL", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CROSS}, + ProductTables: []string{"_raw_" + RAW_WORK_ITEM_TABLE}, +} + +func CollectWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if !data.Options.IsProjectScope() { + return nil + } + logger := taskCtx.GetLogger() + params := data.Options.GetParams() + rawTable := "_raw_" + RAW_WORK_ITEM_TABLE + + stateManager, err := api.NewSubtaskStateManager(&api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_WORK_ITEM_TABLE, + Params: params, + }) + if err != nil { + return err + } + defer stateManager.Close() + + since := "" + if stateManager.IsIncremental() && stateManager.GetSince() != nil { + since = stateManager.GetSince().Format(time.RFC3339) + } + + vsc := azuredevops.NewClient(data.Connection, data.SyncApiClient, data.Connection.GetEndpoint()) + wiql := azuredevops.BuildWiqlQuery(data.Options.ProjectId, since) + ids, err := vsc.QueryWorkItems(azuredevops.QueryWorkItemsArgs{ + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + TeamId: data.Options.TeamId, + Wiql: wiql, + }) + if err != nil { + return err + } + if len(ids) == 0 { + ids, err = collectWorkItemIdsFromIterations(taskCtx, data) + if err != nil { + return err + } + logger.Info("wiql returned no work items, fallback to %d ids from iterations", len(ids)) + } + logger.Info("found %d work item ids", len(ids)) + + db := taskCtx.GetDal() + err = db.AutoMigrate(&api.RawData{}, dal.From(rawTable)) + if err != nil { + return err + } + paramsStr := plugin.MarshalScopeParams(params) + if !stateManager.IsIncremental() { + err = db.Delete(&api.RawData{}, dal.From(rawTable), dal.Where("params = ?", paramsStr)) + if err != nil { + return err + } + } + + totalFetched := 0 + for i := 0; i < len(ids); i += workItemBatchSize { + end := i + workItemBatchSize + if end > len(ids) { + end = len(ids) + } + currentIds := ids[i:end] + items, err := vsc.GetWorkItemsBatch(azuredevops.GetWorkItemsBatchArgs{ + OrgId: data.Options.OrganizationId, + ProjectId: data.Options.ProjectId, + Ids: currentIds, + }) + if err != nil { + return err + } + totalFetched += len(items) + if len(items) == 0 { + logger.Warn(nil, "workitemsbatch returned 0 items for ids range [%d:%d] (size=%d), project=%s team=%s", + i, end, len(currentIds), data.Options.ProjectId, data.Options.TeamId) + } + rows := make([]*api.RawData, len(items)) + for j, item := range items { + body, mErr := json.Marshal(item) + if mErr != nil { + return errors.Convert(mErr) + } + rows[j] = &api.RawData{ + Params: paramsStr, + Data: body, + Url: item.Url, + } + } + if len(rows) > 0 { + err = db.Create(rows, dal.From(rawTable)) + if err != nil { + return err + } + } + } + if len(ids) > 0 && totalFetched == 0 { + return errors.Default.New("workitemsbatch returned no items for all requested ids") + } + return nil +} + +func collectWorkItemIdsFromIterations( + taskCtx plugin.SubTaskContext, + data *AzuredevopsTaskData, +) ([]int, errors.Error) { + db := taskCtx.GetDal() + cursor, err := db.Cursor( + dal.Select("distinct iwi.work_item_id"), + dal.From("_tool_azuredevops_go_iteration_work_items iwi"), + dal.Join("inner join _tool_azuredevops_go_iterations i on i.connection_id = iwi.connection_id and i.iteration_id = iwi.iteration_id"), + dal.Where("i.connection_id = ? and i.team_id = ?", data.Options.ConnectionId, data.Options.TeamId), + ) + if err != nil { + return nil, err + } + defer cursor.Close() + + type workItemIdRow struct { + WorkItemId int `gorm:"column:work_item_id"` + } + var ids []int + for cursor.Next() { + var row workItemIdRow + if err = db.Fetch(cursor, &row); err != nil { + return nil, err + } + if row.WorkItemId > 0 { + ids = append(ids, row.WorkItemId) + } + } + sort.Ints(ids) + return ids, nil +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_converter.go b/backend/plugins/azuredevops_go/tasks/work_item_converter.go new file mode 100644 index 00000000000..161d66c0feb --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_converter.go @@ -0,0 +1,129 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&ConvertWorkItemsMeta) +} + +var ConvertWorkItemsMeta = plugin.SubTaskMeta{ + Name: "convertWorkItems", + EntryPoint: ConvertWorkItems, + EnabledByDefault: true, + Description: "Convert Azure DevOps work items to domain issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{ + models.AzuredevopsWorkItem{}.TableName(), + models.AzuredevopsBoardWorkItem{}.TableName(), + }, + ProductTables: []string{ + ticket.Issue{}.TableName(), + ticket.BoardIssue{}.TableName(), + crossdomain.Account{}.TableName(), + }, +} + +func ConvertWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + db := taskCtx.GetDal() + issueIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsWorkItem{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsUser{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsBoard{}) + boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.Select("_tool_azuredevops_go_work_items.*"), + dal.From("_tool_azuredevops_go_work_items"), + dal.Join(`left join _tool_azuredevops_go_board_work_items bwi + on bwi.work_item_id = _tool_azuredevops_go_work_items.work_item_id + and bwi.connection_id = _tool_azuredevops_go_work_items.connection_id`), + dal.Where("bwi.connection_id = ? AND bwi.team_id = ?", data.Options.ConnectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_WORK_ITEM_TABLE, + }, + InputRowType: reflect.TypeOf(models.AzuredevopsWorkItem{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + wi := inputRow.(*models.AzuredevopsWorkItem) + issueId := issueIdGen.Generate(wi.ConnectionId, wi.WorkItemId) + var result []interface{} + issue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueId}, + IssueKey: wi.IssueKey, + Title: wi.Title, + Type: wi.StdType, + OriginalType: wi.Type, + Status: wi.StdStatus, + OriginalStatus: wi.State, + StoryPoint: wi.StoryPoint, + ResolutionDate: wi.ResolutionDate, + CreatedDate: &wi.CreatedDate, + UpdatedDate: &wi.UpdatedDate, + LeadTimeMinutes: wi.LeadTimeMinutes, + Priority: wi.Priority, + EpicKey: wi.EpicKey, + } + if wi.AssigneeId != "" { + issue.AssigneeId = accountIdGen.Generate(wi.ConnectionId, wi.AssigneeId) + result = append(result, &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{Id: issue.AssigneeId}, + FullName: wi.AssigneeName, + }) + } + if wi.CreatorId != "" { + issue.CreatorId = accountIdGen.Generate(wi.ConnectionId, wi.CreatorId) + result = append(result, &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{Id: issue.CreatorId}, + FullName: wi.CreatorName, + }) + } + result = append(result, issue, &ticket.BoardIssue{ + BoardId: boardId, + IssueId: issueId, + }) + return result, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/work_item_extractor.go b/backend/plugins/azuredevops_go/tasks/work_item_extractor.go new file mode 100644 index 00000000000..f3afa8e9d3b --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/work_item_extractor.go @@ -0,0 +1,142 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&ExtractWorkItemsMeta) +} + +var ExtractWorkItemsMeta = plugin.SubTaskMeta{ + Name: "extractWorkItems", + EntryPoint: ExtractWorkItems, + EnabledByDefault: true, + Description: "Extract Azure DevOps work items from raw data", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{"_raw_" + RAW_WORK_ITEM_TABLE}, + ProductTables: []string{models.AzuredevopsWorkItem{}.TableName()}, +} + +func ExtractWorkItems(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AzuredevopsTaskData) + if !data.Options.IsProjectScope() { + return nil + } + mappings := mergeTypeMappings(data.Options.ScopeConfig) + spField := storyPointField(data.Options.ScopeConfig) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_WORK_ITEM_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiItem azuredevops.WorkItem + if err := json.Unmarshal(row.Data, &apiItem); err != nil { + return nil, errors.Convert(err) + } + return extractWorkItem(data, &apiItem, mappings, spField) + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func extractWorkItem(data *AzuredevopsTaskData, apiItem *azuredevops.WorkItem, mappings map[string]models.TypeMapping, spField string) ([]interface{}, errors.Error) { + fields := apiItem.Fields + wiType := azuredevops.FieldString(fields, "System.WorkItemType") + state := azuredevops.FieldString(fields, "System.State") + stdType, stdStatus := mapWorkItemTypeAndStatus(wiType, state, mappings) + + created, err := parseADOTime(azuredevops.FieldString(fields, "System.CreatedDate")) + if err != nil { + return nil, errors.Convert(err) + } + updated, err := parseADOTime(azuredevops.FieldString(fields, "System.ChangedDate")) + if err != nil { + return nil, errors.Convert(err) + } + var resolutionDate *time.Time + if closed := azuredevops.FieldString(fields, "Microsoft.VSTS.Common.ClosedDate"); closed != "" { + resolutionDate, _ = parseADOTimePtr(closed) + } else if closed := azuredevops.FieldString(fields, "System.ClosedDate"); closed != "" { + resolutionDate, _ = parseADOTimePtr(closed) + } + + workItem := &models.AzuredevopsWorkItem{ + ConnectionId: data.Options.ConnectionId, + WorkItemId: apiItem.Id, + TeamId: data.Options.TeamId, + ProjectId: data.Options.ProjectId, + IssueKey: fmt.Sprintf("%d", apiItem.Id), + Title: azuredevops.FieldString(fields, "System.Title"), + Type: wiType, + State: state, + Priority: azuredevops.FieldString(fields, "Microsoft.VSTS.Common.Priority"), + StoryPoint: azuredevops.FieldFloat(fields, spField), + IterationPath: azuredevops.FieldString(fields, "System.IterationPath"), + CreatorId: azuredevops.FieldIdentityId(fields, "System.CreatedBy"), + CreatorName: azuredevops.FieldString(fields, "System.CreatedBy"), + AssigneeId: azuredevops.FieldIdentityId(fields, "System.AssignedTo"), + AssigneeName: azuredevops.FieldString(fields, "System.AssignedTo"), + CreatedDate: created, + UpdatedDate: updated, + ResolutionDate: resolutionDate, + StdType: stdType, + StdStatus: stdStatus, + } + if parentId := azuredevops.FieldInt(fields, "System.Parent"); parentId > 0 { + workItem.EpicKey = fmt.Sprintf("%d", parentId) + } + if workItem.ResolutionDate != nil { + temp := uint(workItem.ResolutionDate.Sub(workItem.CreatedDate).Minutes()) + workItem.LeadTimeMinutes = &temp + } + + return []interface{}{workItem}, nil +} + +func parseADOTime(s string) (time.Time, error) { + t, err := time.Parse(time.RFC3339, s) + if err != nil { + t, err = time.Parse("2006-01-02T15:04:05.9999999Z", s) + } + return t, err +} + +func parseADOTimePtr(s string) (*time.Time, error) { + t, err := parseADOTime(s) + if err != nil { + return nil, err + } + return &t, nil +} diff --git a/config-ui/src/api/scope/index.ts b/config-ui/src/api/scope/index.ts index 7007314f694..80a4595c0bf 100644 --- a/config-ui/src/api/scope/index.ts +++ b/config-ui/src/api/scope/index.ts @@ -19,12 +19,22 @@ import { IDataScope, IScopeConfig } from '@/types'; import { request } from '@/utils'; +type ScopeResource = 'scopes' | 'repos'; +type RemoteResource = 'remote-scopes' | 'remote-repos'; + +const scopeBase = (plugin: string, connectionId: ID, resource: ScopeResource = 'scopes') => + `/plugins/${plugin}/connections/${connectionId}/${resource}`; + +const remoteBase = (plugin: string, connectionId: ID, resource: RemoteResource = 'remote-scopes') => + `/plugins/${plugin}/connections/${connectionId}/${resource}`; + export const list = ( plugin: string, connectionId: ID, data?: Pagination & { blueprints?: boolean; searchTerm?: string; + scopeResource?: ScopeResource; }, ): Promise<{ count: number; @@ -32,29 +42,46 @@ export const list = ( scope: IDataScope; scopeConfig?: IScopeConfig; }>; -}> => - request(`/plugins/${plugin}/connections/${connectionId}/scopes`, { - data, - }); +}> => { + const { scopeResource = 'scopes', ...query } = data ?? {}; + return request(scopeBase(plugin, connectionId, scopeResource), { data: query }); +}; -export const get = (plugin: string, connectionId: ID, scopeId: ID, payload?: { blueprints: boolean }) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}`, { - data: payload, - }); +export const get = ( + plugin: string, + connectionId: ID, + scopeId: ID, + payload?: { blueprints: boolean; scopeResource?: ScopeResource }, +) => { + const { scopeResource = 'scopes', ...data } = payload ?? {}; + return request(`${scopeBase(plugin, connectionId, scopeResource)}/${scopeId}`, { data }); +}; -export const remove = (plugin: string, connectionId: ID, scopeId: ID, onlyData: boolean) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}?delete_data_only=${onlyData}`, { +export const remove = ( + plugin: string, + connectionId: ID, + scopeId: ID, + onlyData: boolean, + scopeResource: ScopeResource = 'scopes', +) => + request(`${scopeBase(plugin, connectionId, scopeResource)}/${scopeId}?delete_data_only=${onlyData}`, { method: 'delete', }); -export const update = (plugin: string, connectionId: ID, scopeId: ID, payload: any) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes/${scopeId}`, { +export const update = ( + plugin: string, + connectionId: ID, + scopeId: ID, + payload: any, + scopeResource: ScopeResource = 'scopes', +) => + request(`${scopeBase(plugin, connectionId, scopeResource)}/${scopeId}`, { method: 'patch', data: payload, }); -export const batch = (plugin: string, connectionId: ID, payload: any) => - request(`/plugins/${plugin}/connections/${connectionId}/scopes`, { +export const batch = (plugin: string, connectionId: ID, payload: any, scopeResource: ScopeResource = 'scopes') => + request(scopeBase(plugin, connectionId, scopeResource), { method: 'put', data: payload, }); @@ -62,6 +89,7 @@ export const batch = (plugin: string, connectionId: ID, payload: any) => type RemoteQuery = { groupId: ID | null; pageToken?: string; + remoteResource?: RemoteResource; }; type RemoteScope = { @@ -77,22 +105,28 @@ export const remote = ( plugin: string, connectionId: ID, data: RemoteQuery, -): Promise<{ children: RemoteScope[]; nextPageToken: string }> => - request(`/plugins/${plugin}/connections/${connectionId}/remote-scopes`, { +): Promise<{ children: RemoteScope[]; nextPageToken: string }> => { + const { remoteResource = 'remote-scopes', ...query } = data; + return request(remoteBase(plugin, connectionId, remoteResource), { method: 'get', - data, + data: query, }); +}; type SearchRemoteQuery = { search?: string; + remoteResource?: RemoteResource; } & Pagination; export const searchRemote = ( plugin: string, connectionId: ID, data: SearchRemoteQuery, -): Promise<{ children: RemoteScope[]; count: number }> => - request(`/plugins/${plugin}/connections/${connectionId}/search-remote-scopes`, { +): Promise<{ children: RemoteScope[]; count: number }> => { + const resource = data.remoteResource === 'remote-repos' ? 'search-remote-repos' : 'search-remote-scopes'; + const { remoteResource, ...query } = data; + return request(`/plugins/${plugin}/connections/${connectionId}/${resource}`, { method: 'get', - data, + data: query, }); +}; diff --git a/config-ui/src/plugins/components/connection-form/fields/index.tsx b/config-ui/src/plugins/components/connection-form/fields/index.tsx index f6aeb0df9c6..578b0920cf8 100644 --- a/config-ui/src/plugins/components/connection-form/fields/index.tsx +++ b/config-ui/src/plugins/components/connection-form/fields/index.tsx @@ -44,9 +44,9 @@ export const Form = ({ type, name, fields, initialValues, values, errors, setVal const getProps = (key: string, defaultValue: any = '') => { return { name, - initialValue: initialValues[key] ?? defaultValue, - value: values[key] ?? defaultValue, - error: errors[key] ?? defaultValue, + initialValue: initialValues?.[key] ?? defaultValue, + value: values?.[key] ?? defaultValue, + error: errors?.[key] ?? defaultValue, setValue: (value: any) => onValues({ [key]: value }), setError: (value: any) => onErrors({ [key]: value }), }; @@ -57,10 +57,10 @@ export const Form = ({ type, name, fields, initialValues, values, errors, setVal if (typeof field === 'function') { return field({ type, - initialValues, - values, + initialValues: initialValues ?? {}, + values: values ?? {}, setValues: onValues, - errors, + errors: errors ?? {}, setErrors: onErrors, }); } diff --git a/config-ui/src/plugins/components/connection-form/index.tsx b/config-ui/src/plugins/components/connection-form/index.tsx index b61c1d5d076..34232f7fe0d 100644 --- a/config-ui/src/plugins/components/connection-form/index.tsx +++ b/config-ui/src/plugins/components/connection-form/index.tsx @@ -132,7 +132,7 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { connectionId, buildUpdateTestPayload(connection, values, sanitizedCustomHeaders), ) - : API.connection.testOld(plugin, buildCreateTestPayload(initialValues, values, sanitizedCustomHeaders)), + : API.connection.testOld(plugin, buildCreateTestPayload(initialValues ?? {}, values, sanitizedCustomHeaders)), { setOperating, formatMessage: () => 'Test Connection Successfully.', @@ -175,7 +175,7 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { type={type} name={name} fields={fields} - initialValues={connection ? { ...initialValues, ...connection } : initialValues} + initialValues={connection ? { ...(initialValues ?? {}), ...connection } : (initialValues ?? {})} values={values} errors={errors} setValues={setValues} diff --git a/config-ui/src/plugins/components/connection-list/index.tsx b/config-ui/src/plugins/components/connection-list/index.tsx index e0985b1d14d..2b253f25eb1 100644 --- a/config-ui/src/plugins/components/connection-list/index.tsx +++ b/config-ui/src/plugins/components/connection-list/index.tsx @@ -113,7 +113,7 @@ export const ConnectionList = ({ plugin, onCreate }: Props) => { dataSource={connections} pagination={false} /> - void; @@ -42,6 +44,7 @@ export const DataScopeRemote = ({ mode = 'multiple', plugin, connectionId, + dataScopeConfig, disabledScope, onChangeSelectedScope, footer, @@ -56,11 +59,20 @@ export const DataScopeRemote = ({ setSelectedScope(props.selectedScope ?? []); }, [props.selectedScope]); - const config = useMemo(() => getPluginConfig(plugin).dataScope, [plugin]); + const config = useMemo( + () => dataScopeConfig ?? getPluginConfig(plugin).dataScope, + [plugin, dataScopeConfig], + ); const handleSubmit = async () => { const [success, res] = await operator( - () => API.scope.batch(plugin, connectionId, { data: selectedScope.map((it) => it.data) }), + () => + API.scope.batch( + plugin, + connectionId, + { data: selectedScope.map((it) => it.data) }, + config.scopeResource ?? 'scopes', + ), { setOperating, formatMessage: () => 'Add data scope successful.', diff --git a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx index eba655a1f4f..5cc7fdc509f 100644 --- a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx +++ b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx @@ -100,6 +100,7 @@ export const SearchLocal = ({ mode, plugin, connectionId, config, disabledScope, const res = await API.scope.remote(plugin, connectionId, { groupId, pageToken: currentPageToken, + remoteResource: config.remoteResource, }); newItems = (res.children ?? []).map((it) => ({ diff --git a/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx b/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx index 5f61d74106b..43f6c474eb4 100644 --- a/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx +++ b/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx @@ -94,6 +94,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope const res = await API.scope.remote(plugin, connectionId, { groupId, pageToken: currentPageToken, + remoteResource: config.remoteResource, }); newItems = (res.children ?? []).map((it) => ({ @@ -138,6 +139,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope search: searchDebounce, page: search.page, pageSize: PAGE_SIZE, + remoteResource: config.remoteResource, }); const newItems = (res.children ?? []).map((it) => ({ diff --git a/config-ui/src/plugins/components/data-scope-select/index.tsx b/config-ui/src/plugins/components/data-scope-select/index.tsx index 860cce6cd06..c474a93ef9e 100644 --- a/config-ui/src/plugins/components/data-scope-select/index.tsx +++ b/config-ui/src/plugins/components/data-scope-select/index.tsx @@ -27,7 +27,7 @@ import API from '@/api'; import { PATHS } from '@/config'; import { Loading, Block, ExternalLink, Message } from '@/components'; import { useRefreshData } from '@/hooks'; -import { getPluginScopeId, getPluginScopeName } from '@/plugins'; +import { getPluginConfig, getPluginScopeId, getPluginScopeName } from '@/plugins'; interface Props { plugin: string; @@ -55,6 +55,29 @@ export const DataScopeSelect = ({ const [pageSize] = useState(10); const [total, setTotal] = useState(0); + const pluginConfig = useMemo(() => getPluginConfig(plugin), [plugin]); + + const scopeResources = useMemo(() => { + const resources: Array<'scopes' | 'repos'> = []; + const primary = pluginConfig.dataScope?.scopeResource ?? 'scopes'; + resources.push(primary); + const secondary = pluginConfig.secondaryDataScope?.scopeResource; + if (secondary && secondary !== primary) { + resources.push(secondary); + } + return resources; + }, [pluginConfig]); + + const toScopeItems = (scopes: any[], prefix = '') => + scopes + .filter((sc) => sc.scope) + .map((sc) => ({ + parentId: null, + id: getPluginScopeId(plugin, sc.scope), + title: `${prefix}${getPluginScopeName(plugin, sc.scope) || sc.scope?.fullName || sc.scope?.name || sc.scope?.id}`, + data: sc.scope, + })); + useEffect(() => { setSelectedIds((initialScope ?? []).map((sc) => sc.id)); }, []); @@ -64,18 +87,24 @@ export const DataScopeSelect = ({ setLoading(true); } - const res = await API.scope.list(plugin, connectionId, { page, pageSize }); - setItems((items) => [ - ...items, - ...res.scopes.map((sc) => ({ - parentId: null, - id: getPluginScopeId(plugin, sc.scope), - title: getPluginScopeName(plugin, sc.scope) || sc.scope.fullName || sc.scope.name, - data: sc.scope, - })), - ]); - - setTotal(res.count); + const responses = await Promise.all( + scopeResources.map((scopeResource) => + API.scope.list(plugin, connectionId, { page, pageSize, scopeResource }), + ), + ); + + const newItems = responses.flatMap((res, index) => { + const prefix = + scopeResources.length > 1 + ? scopeResources[index] === 'repos' + ? '[Repo] ' + : '[Board] ' + : ''; + return toScopeItems(res.scopes, prefix); + }); + + setItems((items) => (page === 1 ? newItems : [...items, ...newItems])); + setTotal(responses.reduce((sum, res) => sum + res.count, 0)); setLoading(false); }; @@ -86,17 +115,40 @@ export const DataScopeSelect = ({ const search = useDebounce(query, { wait: 500 }); const { ready, data } = useRefreshData( - async () => await API.scope.list(plugin, connectionId, { searchTerm: search }), - [search], + async () => { + const responses = await Promise.all( + scopeResources.map((scopeResource) => + API.scope.list(plugin, connectionId, { searchTerm: search, scopeResource }), + ), + ); + return { + scopes: responses.flatMap((res, index) => { + const prefix = + scopeResources.length > 1 + ? scopeResources[index] === 'repos' + ? '[Repo] ' + : '[Board] ' + : ''; + return res.scopes.map((sc) => ({ + ...sc, + scope: sc.scope, + _label: `${prefix}${getPluginScopeName(plugin, sc.scope) || sc.scope?.fullName || sc.scope?.name}`, + })); + }), + }; + }, + [search, plugin, connectionId, scopeResources], ); const searchOptions = useMemo( () => - data?.scopes.map((sc) => ({ - label: getPluginScopeName(plugin, sc.scope) || sc.scope.fullName || sc.scope.name, + data?.scopes.map((sc: any) => ({ + label: + sc._label ?? + (getPluginScopeName(plugin, sc.scope) || sc.scope?.fullName || sc.scope?.name), value: getPluginScopeId(plugin, sc.scope), })) ?? [], - [data], + [data, plugin], ); const handleScroll = () => setPage(page + 1); diff --git a/config-ui/src/plugins/register/azure/config.tsx b/config-ui/src/plugins/register/azure/config.tsx index 4d726f1ec26..993e4de9eee 100644 --- a/config-ui/src/plugins/register/azure/config.tsx +++ b/config-ui/src/plugins/register/azure/config.tsx @@ -82,19 +82,31 @@ export const AzureGoConfig: IPluginConfig = { isBeta: true, connection: { docLink: DOC_URL.PLUGIN.AZUREDEVOPS.BASIS, + initialValues: { + rateLimitPerHour: 18000, + }, fields: [ 'name', () => , { key: 'token', label: 'Personal Access Token', + subLabel: ( + + Required scopes: Code (read), Build (read), and{' '} + Work Items (read) (vso.work). + + ), }, ({ initialValues, values, setValues }: any) => ( setValues({ organization: value })} /> ), @@ -112,12 +124,23 @@ export const AzureGoConfig: IPluginConfig = { dataScope: { localSearch: true, title: 'Repositories', + scopeResource: 'repos', + remoteResource: 'remote-repos', + millerColumn: { + columnCount: 2, + }, + }, + secondaryDataScope: { + localSearch: true, + title: 'Boards / Teams', + scopeResource: 'scopes', + remoteResource: 'remote-scopes', millerColumn: { columnCount: 2, }, }, scopeConfig: { - entities: ['CODE', 'CODEREVIEW', 'CROSS', 'CICD'], + entities: ['CODE', 'CODEREVIEW', 'CROSS', 'CICD', 'TICKET'], transformation: { deploymentPattern: '(deploy|push-image)', productionPattern: 'prod(.*)', @@ -125,6 +148,15 @@ export const AzureGoConfig: IPluginConfig = { tagsLimit: 10, tagsPattern: '/v\\d+\\.\\d+(\\.\\d+(-rc)*\\d*)*$/', }, + typeMappings: { + 'User Story': { standardType: 'REQUIREMENT' }, + 'Product Backlog Item': { standardType: 'REQUIREMENT' }, + Feature: { standardType: 'REQUIREMENT' }, + Epic: { standardType: 'REQUIREMENT' }, + Bug: { standardType: 'BUG' }, + 'Production Incident': { standardType: 'INCIDENT' }, + }, + storyPointField: 'Microsoft.VSTS.Scheduling.StoryPoints', }, }, }; diff --git a/config-ui/src/plugins/register/azure/connection-fields/organization.tsx b/config-ui/src/plugins/register/azure/connection-fields/organization.tsx index 1e2972ca382..80f345e9c2e 100644 --- a/config-ui/src/plugins/register/azure/connection-fields/organization.tsx +++ b/config-ui/src/plugins/register/azure/connection-fields/organization.tsx @@ -38,11 +38,11 @@ export const ConnectionOrganization = ({ label, initialValue, value, setValue }: const [settings, setSettings] = useState({ scoped: false, organization: '' }); useEffect(() => { - const org = initialValue.organization || ''; + const org = initialValue?.organization || ''; setValue(org); - setSettings({ organization: initialValue.organization, scoped: org !== '' }); - }, [initialValue.organization]); + setSettings({ organization: initialValue?.organization ?? '', scoped: org !== '' }); + }, [initialValue?.organization]); const handleChange = (e: RadioChangeEvent) => { const scoped = e.target.value; diff --git a/config-ui/src/plugins/register/azure/transformation.tsx b/config-ui/src/plugins/register/azure/transformation.tsx index 1a8ea080740..0c73b3ac4ee 100644 --- a/config-ui/src/plugins/register/azure/transformation.tsx +++ b/config-ui/src/plugins/register/azure/transformation.tsx @@ -16,19 +16,63 @@ * */ +import { useState, useEffect } from 'react'; import { CaretRightOutlined } from '@ant-design/icons'; -import { theme, Collapse, Tag, Input } from 'antd'; +import { theme, Collapse, Tag, Input, Form, Select } from 'antd'; import { ExternalLink, HelpTooltip } from '@/components'; import { DOC_URL } from '@/release'; +enum StandardType { + Requirement = 'REQUIREMENT', + Bug = 'BUG', + Incident = 'INCIDENT', +} + +const ADO_WORK_ITEM_TYPES = [ + 'User Story', + 'Product Backlog Item', + 'Feature', + 'Epic', + 'Bug', + 'Task', + 'Production Incident', +]; + +const ADO_STORY_POINT_FIELDS = [ + { label: 'Story Points', value: 'Microsoft.VSTS.Scheduling.StoryPoints' }, + { label: 'Effort', value: 'Microsoft.VSTS.Scheduling.Effort' }, +]; + interface Props { entities: string[]; + connectionId?: ID; transformation: any; setTransformation: React.Dispatch>; } export const AzureTransformation = ({ entities, transformation, setTransformation }: Props) => { + const [requirements, setRequirements] = useState([]); + const [bugs, setBugs] = useState([]); + const [incidents, setIncidents] = useState([]); + + useEffect(() => { + const types = Object.entries(transformation.typeMappings ?? {}).map(([key, value]: any) => ({ + name: key, + ...value, + })); + + setRequirements(types.filter((it) => it.standardType === StandardType.Requirement).map((it) => it.name)); + setBugs(types.filter((it) => it.standardType === StandardType.Bug).map((it) => it.name)); + setIncidents(types.filter((it) => it.standardType === StandardType.Incident).map((it) => it.name)); + }, [transformation]); + + const transformType = (its: string[], standardType: StandardType) => + its.reduce((acc, cur) => { + acc[cur] = { standardType }; + return acc; + }, {} as any); + const { token } = theme.useToken(); const panelStyle: React.CSSProperties = { @@ -41,7 +85,7 @@ export const AzureTransformation = ({ entities, transformation, setTransformatio return ( } style={{ background: token.colorBgContainer }} size="large" @@ -50,6 +94,10 @@ export const AzureTransformation = ({ entities, transformation, setTransformatio panelStyle, transformation, onChangeTransformation: setTransformation, + requirements, + bugs, + incidents, + transformType, })} /> ); @@ -60,13 +108,97 @@ const renderCollapseItems = ({ panelStyle, transformation, onChangeTransformation, + requirements, + bugs, + incidents, + transformType, }: { entities: string[]; panelStyle: React.CSSProperties; transformation: any; onChangeTransformation: any; + requirements: string[]; + bugs: string[]; + incidents: string[]; + transformType: (its: string[], standardType: StandardType) => Record; }) => [ + { + key: 'TICKET', + label: 'Issue Tracking', + style: panelStyle, + children: ( +
+

+ Map Azure DevOps work item types to DevLake standard types (REQUIREMENT, BUG, INCIDENT) and choose the + story points field. +

+ + ({ label: it, value: it }))} + value={bugs} + onChange={(value) => + onChangeTransformation({ + ...transformation, + typeMappings: { + ...transformType(requirements, StandardType.Requirement), + ...transformType(value, StandardType.Bug), + ...transformType(incidents, StandardType.Incident), + }, + }) + } + /> + + + + onChangeTransformation({ + ...transformation, + storyPointField: value, + }) + } + /> + +
+ ), + }, { key: 'CICD', label: 'CI/CD', diff --git a/config-ui/src/plugins/utils.ts b/config-ui/src/plugins/utils.ts index 34c6ed63b14..dd571eecf56 100644 --- a/config-ui/src/plugins/utils.ts +++ b/config-ui/src/plugins/utils.ts @@ -47,6 +47,8 @@ export const getPluginScopeId = (plugin: string, scope: any) => { return `${scope.gid}`; case 'linear': return `${scope.teamId}`; + case 'azuredevops_go': + return `${scope.teamId ?? scope.id}`; default: return `${scope.id}`; } diff --git a/config-ui/src/routes/connection/connection.tsx b/config-ui/src/routes/connection/connection.tsx index 7f21a7ff5f8..25f83af5266 100644 --- a/config-ui/src/routes/connection/connection.tsx +++ b/config-ui/src/routes/connection/connection.tsx @@ -23,10 +23,11 @@ import { DeleteOutlined, PlusOutlined, LinkOutlined, ClearOutlined } from '@ant- import { theme, Space, Table, Button, Modal, message } from 'antd'; import API from '@/api'; -import { PageHeader, Message, IconButton } from '@/components'; +import { PageHeader, Message, IconButton, PageLoading } from '@/components'; import { PATHS } from '@/config'; import { useAppDispatch, useAppSelector } from '@/hooks'; import { selectConnection, removeConnection } from '@/features'; +import { transformConnection } from '@/features/connections/utils'; import { useRefreshData } from '@/hooks'; import { ConnectionStatus, @@ -48,6 +49,7 @@ export const Connection = () => { const [type, setType] = useState< | 'deleteConnection' | 'createDataScope' + | 'createSecondaryDataScope' | 'clearDataScope' | 'deleteDataScope' | 'associateScopeConfig' @@ -60,7 +62,9 @@ export const Connection = () => { const [version, setVersion] = useState(1); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(10); + const [boardPage, setBoardPage] = useState(1); const [scopeId, setScopeId] = useState(); + const [scopeResource, setScopeResource] = useState<'scopes' | 'repos'>('scopes'); const [scopeIds, setScopeIds] = useState([]); const [conflict, setConflict] = useState([]); const [errorMsg, setErrorMsg] = useState(''); @@ -76,31 +80,71 @@ export const Connection = () => { } = theme.useToken(); const dispatch = useAppDispatch(); - const connection = useAppSelector((state) => selectConnection(state, `${plugin}-${connectionId}`)) as IConnection; + const connectionFromStore = useAppSelector((state) => + selectConnection(state, `${plugin}-${connectionId}`), + ) as IConnection | undefined; const navigate = useNavigate(); - const { ready, data } = useRefreshData( - () => API.scope.list(plugin, connectionId, { page, pageSize, blueprints: true }), - [version, page, pageSize], + const { ready: connectionReady, data: connection } = useRefreshData( + async () => { + if (connectionFromStore) { + return connectionFromStore; + } + const res = await API.connection.get(plugin, connectionId); + return transformConnection(plugin, res); + }, + [plugin, connectionId, connectionFromStore?.unique], ); - const { name } = connection; - const pluginConfig = useMemo(() => getPluginConfig(plugin), [plugin]); + const primaryScopeResource = pluginConfig.dataScope?.scopeResource ?? 'scopes'; + const secondaryScopeResource = pluginConfig.secondaryDataScope?.scopeResource ?? 'scopes'; - const [dataSource, total] = useMemo( - () => [ - data?.scopes.map((it: any) => ({ + const { ready, data } = useRefreshData( + () => + API.scope.list(plugin, connectionId, { + page, + pageSize, + blueprints: true, + scopeResource: primaryScopeResource, + }), + [version, page, pageSize, primaryScopeResource], + ); + + const { ready: boardReady, data: boardData } = useRefreshData( + () => + pluginConfig.secondaryDataScope + ? API.scope.list(plugin, connectionId, { + page: boardPage, + pageSize, + blueprints: true, + scopeResource: secondaryScopeResource, + }) + : Promise.resolve({ scopes: [], count: 0 }), + [version, boardPage, pageSize, pluginConfig.secondaryDataScope, secondaryScopeResource], + ); + + const mapScopes = (scopes: any[] | undefined, resource: 'scopes' | 'repos') => + scopes + ?.filter((it: any) => it.scope) + .map((it: any) => ({ id: getPluginScopeId(plugin, it.scope), - name: getPluginScopeName(plugin, it.scope) || it.scope.fullName || it.scope.name, + name: getPluginScopeName(plugin, it.scope) || it.scope?.fullName || it.scope?.name || 'Unknown', projects: it.blueprints?.map((bp: any) => bp.projectName) ?? [], configId: it.scopeConfig?.id, configName: it.scopeConfig?.name, - })) ?? [], - data?.count ?? 0, - ], - [data], + scopeResource: resource, + })) ?? []; + + const [dataSource, total] = useMemo( + () => [mapScopes(data?.scopes, primaryScopeResource), data?.count ?? 0], + [data, plugin, primaryScopeResource], + ); + + const [boardDataSource, boardTotal] = useMemo( + () => [mapScopes(boardData?.scopes, secondaryScopeResource), boardData?.count ?? 0], + [boardData, plugin, secondaryScopeResource], ); const handleHideDialog = () => { @@ -146,22 +190,30 @@ export const Connection = () => { }; const handleShowCreateDataScopeDialog = () => { + setScopeResource(primaryScopeResource); setType('createDataScope'); }; + const handleShowCreateSecondaryDataScopeDialog = () => { + setScopeResource(secondaryScopeResource); + setType('createSecondaryDataScope'); + }; + const handleCreateDataScope = () => { setVersion((v) => v + 1); handleHideDialog(); }; - const handleShowClearDataScopeDialog = (scopeId: ID) => { + const handleShowClearDataScopeDialog = (id: ID, resource: 'scopes' | 'repos' = primaryScopeResource) => { setType('clearDataScope'); - setScopeId(scopeId); + setScopeId(id); + setScopeResource(resource); }; - const handleShowDeleteDataScopeDialog = (scopeId: ID) => { + const handleShowDeleteDataScopeDialog = (id: ID, resource: 'scopes' | 'repos' = primaryScopeResource) => { setType('deleteDataScope'); - setScopeId(scopeId); + setScopeId(id); + setScopeResource(resource); }; const handleDeleteDataScope = async (onlyData: boolean) => { @@ -170,7 +222,7 @@ export const Connection = () => { const [, res] = await operator( async () => { try { - await API.scope.remove(plugin, connectionId, scopeId, onlyData); + await API.scope.remove(plugin, connectionId, scopeId, onlyData, scopeResource); return { status: 'success' }; } catch (err: any) { const { status, data } = err.response; @@ -225,7 +277,7 @@ export const Connection = () => { await Promise.all( scopeIds.map(async (id) => { try { - await API.scope.remove(plugin, connectionId, id, false); + await API.scope.remove(plugin, connectionId, id, false, primaryScopeResource); successCount++; } catch (err: any) { const scopeName = scopeMap.get(String(id)) || 'Unknown'; @@ -254,11 +306,14 @@ export const Connection = () => { () => Promise.all( scopeIds.map(async (scopeId) => { - const scope = await API.scope.get(plugin, connectionId, scopeId); + const scope = await API.scope.get(plugin, connectionId, scopeId, { + blueprints: true, + scopeResource: primaryScopeResource, + }); return API.scope.update(plugin, connectionId, scopeId, { ...scope, scopeConfigId: trId !== 'None' ? +trId : null, - }); + }, primaryScopeResource); }), ), { @@ -277,6 +332,12 @@ export const Connection = () => { } }; + if (!connectionReady || !connection) { + return ; + } + + const { name } = connection; + return ( {
Please note: In order to view DORA metrics, you will need to add Scope Configs.
+ {pluginConfig.secondaryDataScope && ( + + )} {plugin !== 'tapd' && pluginConfig.scopeConfig && (