finish

Author: warren830

backend/Makefile

@@ -63,7 +63,7 @@ build: build-plugin build-server
 run:
 	go run server/main.go
 
-dev: build-plugin build-python run
+dev: build-plugin run
 
 godev:
 	DISABLED_REMOTE_PLUGINS=true make build-plugin run

backend/plugins/q_dev/api/blueprint_v200.go

@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/plugin"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+	"github.com/apache/incubator-devlake/plugins/q_dev/models"
+	"github.com/apache/incubator-devlake/plugins/q_dev/tasks"
+)
+
+func MakeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	connectionId uint64,
+	bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	// load connection and scope from the db
+	connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopes, err := makeScopesV200(scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig],
+	connection *models.QDevConnection,
+) (coreModels.PipelinePlan, errors.Error) {
+	plan := make(coreModels.PipelinePlan, len(scopeDetails))
+	for i, scopeDetail := range scopeDetails {
+		s3Slice := scopeDetail.Scope
+		stage := plan[i]
+		if stage == nil {
+			stage = coreModels.PipelineStage{}
+		}
+
+		// construct task options for q_dev
+		op := &tasks.QDevOptions{
+			ConnectionId: s3Slice.ConnectionId,
+			S3Prefix:     s3Slice.Prefix,
+		}
+
+		// Pass empty entities array to enable all subtasks
+		task, err := helper.MakePipelinePlanTask("q_dev", subtaskMetas, []string{}, op)
+		if err != nil {
+			return nil, err
+		}
+		stage = append(stage, task)
+		plan[i] = stage
+	}
+	return plan, nil
+}
+
+func makeScopesV200(
+	scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig],
+	connection *models.QDevConnection,
+) ([]plugin.Scope, errors.Error) {
+	scopes := make([]plugin.Scope, 0)
+	// For Q Developer metrics, we don't need to create domain layer scopes
+	// The data is collected and stored directly in the tool layer
+	return scopes, nil
+}

backend/plugins/q_dev/impl/impl.go

@@ -23,6 +23,7 @@ import (
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/q_dev/api"
@@ -39,6 +40,7 @@ var _ interface {
 	plugin.PluginModel
 	plugin.PluginSource
 	plugin.PluginMigration
+	plugin.DataSourcePluginBlueprintV200
 	plugin.CloseablePluginTask
 } = (*QDev)(nil)
 
@@ -170,3 +172,10 @@ func (p QDev) Close(taskCtx plugin.TaskContext) errors.Error {
 	data.S3Client.Close()
 	return nil
 }
+
+func (p QDev) MakeDataSourcePipelinePlanV200(
+	connectionId uint64,
+	scopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+}

backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go

@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+)
+
+type addScopeConfigIdToS3Slice struct{}
+
+func (*addScopeConfigIdToS3Slice) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+
+	// Add scope_config_id column to _tool_q_dev_s3_slices table
+	err := db.Exec(`
+		ALTER TABLE _tool_q_dev_s3_slices
+		ADD COLUMN scope_config_id BIGINT UNSIGNED DEFAULT 0
+	`)
+	if err != nil {
+		return errors.Convert(err)
+	}
+
+	return nil
+}
+
+func (*addScopeConfigIdToS3Slice) Version() uint64 {
+	return 20251123000001
+}
+
+func (*addScopeConfigIdToS3Slice) Name() string {
+	return "Add scope_config_id column to S3 slice table"
+}

backend/plugins/q_dev/models/migrationscripts/register.go

@@ -29,5 +29,6 @@ func All() []plugin.MigrationScript {
 		new(addDisplayNameFields),
 		new(addMissingMetrics),
 		new(addS3SliceTable),
+		new(addScopeConfigIdToS3Slice),
 	}
 }

backend/plugins/q_dev/tasks/identity_client.go

@@ -70,6 +70,11 @@ func NewQDevIdentityClient(connection *models.QDevConnection) (*QDevIdentityClie
 // ResolveUserDisplayName resolves a user ID to a human-readable display name
 // Returns the display name if found, otherwise returns the original userId as fallback
 func (client *QDevIdentityClient) ResolveUserDisplayName(userId string) (string, error) {
+	// Check if client or IdentityStore is nil
+	if client == nil || client.IdentityStore == nil {
+		return userId, nil
+	}
+
 	input := &identitystore.DescribeUserInput{
 		IdentityStoreId: aws.String(client.StoreId),
 		UserId:          aws.String(userId),

backend/plugins/q_dev/tasks/s3_data_extractor.go

@@ -318,4 +318,5 @@ var ExtractQDevS3DataMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Extract data from S3 CSV files and save to database",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CROSS},
+	Dependencies:     []*plugin.SubTaskMeta{&CollectQDevS3FilesMeta},
 }

backend/plugins/q_dev/tasks/s3_file_collector.go

@@ -114,4 +114,5 @@ var CollectQDevS3FilesMeta = plugin.SubTaskMeta{
 	EntryPoint:       CollectQDevS3Files,
 	EnabledByDefault: true,
 	Description:      "Collect S3 file metadata from AWS S3 bucket",
+	DomainTypes:      []string{plugin.DOMAIN_TYPE_CROSS},
 }