From 2171dbf1fc2a7a55e81f5c9f8401672fc5f56fdd Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 08:53:08 +0000 Subject: [PATCH 01/27] ce feat: implement unmasking workflow use cases and data structures Added new files for unmasking workflow functionality, including use cases for creating, retrieving, and managing unmasking workflows. Introduced data structures for unmasking workflows, SQL content, and related operations, ensuring compatibility with both DMS and non-DMS builds. This implementation lays the groundwork for enhanced data masking capabilities. --- .../data_masking/biz/unmasking_workflow.go | 271 ++++++++++++++++++ .../data_masking/biz/unmasking_workflow_ce.go | 84 ++++++ 2 files changed, 355 insertions(+) create mode 100644 internal/data_masking/biz/unmasking_workflow.go create mode 100644 internal/data_masking/biz/unmasking_workflow_ce.go diff --git a/internal/data_masking/biz/unmasking_workflow.go b/internal/data_masking/biz/unmasking_workflow.go new file mode 100644 index 00000000..27b5f0ad --- /dev/null +++ b/internal/data_masking/biz/unmasking_workflow.go @@ -0,0 +1,271 @@ +package biz + +import ( + "context" + "time" + + maskingCore "github.com/actiontech/dms/internal/data_masking/core" +) + +// MaskingConfigStatus 脱敏配置状态 +// swagger:enum MaskingConfigStatus +type MaskingConfigStatus string + +const ( + MaskingConfigStatusPendingConfirm MaskingConfigStatus = "PENDING_CONFIRM" // 系统发现，待人工确认 + MaskingConfigStatusConfigured MaskingConfigStatus = "CONFIGURED" // 用户已确认/手动配置 + MaskingConfigStatusSystemConfirmed MaskingConfigStatus = "SYSTEM_CONFIRMED" // 系统已确认 +) + +// ColumnMaskingConfig 列级别脱敏配置领域模型 +// swagger:model ColumnMaskingConfig +type ColumnMaskingConfig struct { + // 配置记录 ID + ID uint `json:"id"` + // 数据源 UID + DBServiceUID string `json:"db_service_uid"` + // 列 ID（db_columns.id） + ColumnID uint `json:"column_id"` + // Schema 名称 + SchemaName string `json:"schema_name"` + // 表名 + TableName string `json:"table_name"` + // 列名 + ColumnName string `json:"column_name"` + // 是否启用脱敏 + IsMaskingEnabled bool `json:"is_masking_enabled"` + // 脱敏规则 ID + MaskingRuleID int `json:"masking_rule_id"` + // 脱敏规则名称（中文） + MaskingRuleName string `json:"masking_rule_name"` + // 置信度 + Confidence maskingCore.Confidence `json:"confidence,omitempty"` + // 配置状态 + Status MaskingConfigStatus `json:"status"` + // 创建时间 + CreatedAt time.Time `json:"created_at"` + // 更新时间 + UpdatedAt time.Time `json:"updated_at"` +} + +// swagger:model TableRef +type TableRef struct { + // Schema 名称 + Schema string `json:"schema"` + // 表名 + Table string `json:"table"` + // 表别名 + Alias string `json:"alias"` +} + +// swagger:model ColumnRef +type ColumnRef struct { + // Schema 名称 + Schema string `json:"schema"` + // 表名 + Table string `json:"table"` + // 列名 + Column string `json:"column"` +} + +// swagger:model ResultColumn +type ResultColumn struct { + // 结果列名 + Name string `json:"name"` + // 结果列表达式（SQL 片段） + Expression string `json:"expression"` + // 来源列列表 + Sources []ColumnRef `json:"sources"` +} + +// swagger:model LineageNode +type LineageNode struct { + // 节点 ID（图内唯一） + ID string `json:"id"` + // 节点类型 + Type NodeType `json:"type"` + // 节点展示名 + Name string `json:"name"` + // Schema 名称（列节点可能存在） + Schema string `json:"schema"` + // 表名（列节点可能存在） + Table string `json:"table"` + // 列名（列节点可能存在） + Column string `json:"column"` + // 表达式内容（表达式节点可能存在） + Expr string `json:"expr"` +} + +// swagger:model LineageEdge +type LineageEdge struct { + // 起点节点 ID + FromID string `json:"from_id"` + // 终点节点 ID + ToID string `json:"to_id"` + // 边类型 + Type EdgeType `json:"type"` +} + +// swagger:model AnalyzeResult +type AnalyzeResult struct { + // 分析标题/摘要 + Title string `json:"title"` + // 原始 SQL + OriginalSQL string `json:"original_sql"` + // 解析到的表引用列表 + Tables []TableRef `json:"tables"` + // 解析到的源列列表 + SourceColumns []ColumnRef `json:"source_columns"` + // 解析到的结果列列表 + ResultColumns []ResultColumn `json:"result_columns"` + // 血缘图节点 + Nodes []LineageNode `json:"nodes"` + // 血缘图边 + Edges []LineageEdge `json:"edges"` + // 警告信息（解析不完整等） + Warnings []string `json:"warnings,omitempty"` +} + +// NodeType 血缘节点类型 +// swagger:enum NodeType +type NodeType string + +const ( + // 源列节点 + NodeTypeSource NodeType = "source_column" + // 表达式节点 + NodeTypeExpression NodeType = "expression" + // 结果列节点 + NodeTypeResult NodeType = "result_column" + // 表节点 + NodeTypeTable NodeType = "table" +) + +// EdgeType 血缘边类型 +// swagger:enum EdgeType +type EdgeType string + +const ( + // 直接依赖 + EdgeTypeDirect EdgeType = "direct" + // 转换/计算依赖 + EdgeTypeTransform EdgeType = "transform" + // 聚合依赖 + EdgeTypeAggregate EdgeType = "aggregate" +) + +// UnmaskingWorkflowApprovalStatus 审批状态 +// swagger:enum UnmaskingWorkflowApprovalStatus +type UnmaskingWorkflowApprovalStatus string + +const ( + // 待审批 + UnmaskingWorkflowApprovalStatusPending UnmaskingWorkflowApprovalStatus = "pending" + // 已批准 + UnmaskingWorkflowApprovalStatusApproved UnmaskingWorkflowApprovalStatus = "approved" + // 已驳回 + UnmaskingWorkflowApprovalStatusRejected UnmaskingWorkflowApprovalStatus = "rejected" + // 已取消 + UnmaskingWorkflowApprovalStatusCancelled UnmaskingWorkflowApprovalStatus = "cancelled" +) + +func (s UnmaskingWorkflowApprovalStatus) String() string { + return string(s) +} + +// UnmaskingWorkflowUsageStatus 使用情况 +// swagger:enum UnmaskingWorkflowUsageStatus +type UnmaskingWorkflowUsageStatus string + +const ( + // 未查看 + UnmaskingWorkflowUsageStatusUnviewed UnmaskingWorkflowUsageStatus = "unviewed" + // 已查看 + UnmaskingWorkflowUsageStatusViewed UnmaskingWorkflowUsageStatus = "viewed" +) + +func (s UnmaskingWorkflowUsageStatus) String() string { + return string(s) +} + +// UnmaskingWorkflowSourceType 来源类型 +// swagger:enum UnmaskingWorkflowSourceType +type UnmaskingWorkflowSourceType string + +const ( + // 数据导出工单 + UnmaskingWorkflowSourceTypeDataExport UnmaskingWorkflowSourceType = "data_export" + // SQL工作台 + UnmaskingWorkflowSourceTypeSQLWorkbench UnmaskingWorkflowSourceType = "sql_workbench" +) + +func (s UnmaskingWorkflowSourceType) String() string { + return string(s) +} + +// UnmaskingOriginalDataCredentialArgs 工单凭证路径参数，供 CheckOriginalDataAccess 使用；nil 表示仅做权限直通（不入库解析工单）。enterprise 与 dms 构建共用。 +type UnmaskingOriginalDataCredentialArgs struct { + SourceType UnmaskingWorkflowSourceType + SourceUID string + Credential string +} + +// UnmaskingAction 操作动作类型 +// swagger:enum UnmaskingAction +type UnmaskingAction string + +const ( + // 提交申请 + UnmaskingActionSubmit UnmaskingAction = "submit" + // 批准申请 + UnmaskingActionApprove UnmaskingAction = "approve" + // 驳回申请 + UnmaskingActionReject UnmaskingAction = "reject" + // 查看工单详情 + UnmaskingActionViewOriginalDataWorkflowDetail UnmaskingAction = "view_unmasking_workflow_detail" + // 查看原文 + UnmaskingActionViewOriginalData UnmaskingAction = "view_full_original_data" + // 下载原文 + UnmaskingActionDownloadOriginalData UnmaskingAction = "download_full_original_data" + // 取消申请 + UnmaskingActionCancel UnmaskingAction = "cancel" +) + +func (a UnmaskingAction) String() string { + return string(a) +} + +// 与 DMS OpRangeType 字符串取值一致，用于 UnmaskingOpPermissionRange 判定。 +const ( + UnmaskingOpRangeProject = "project" + UnmaskingOpRangeDBService = "db_service" +) + +// UnmaskingOpPermissionRange 查看原文工单权限判定用的「操作权限 + 范围」快照。 +type UnmaskingOpPermissionRange struct { + OpPermissionUID string + OpRangeType string + RangeUIDs []string +} + +// UnmaskingWorkflowOpPermissionVerifier 操作权限校验能力。 +type UnmaskingWorkflowOpPermissionVerifier interface { + IsUserDMSAdmin(ctx context.Context, userUID string) (bool, error) + CanOpGlobal(ctx context.Context, userUID string) (bool, error) + CanViewGlobal(ctx context.Context, userUID string) (bool, error) + IsUserProjectAdmin(ctx context.Context, userUID, projectUID string) (bool, error) + GetUserOpPermissionInProject(ctx context.Context, userUID, projectUID string) ([]UnmaskingOpPermissionRange, error) + GetCanOpDBUsers(ctx context.Context, projectUID, dbServiceUID string, needOpPermissionTypes []string) ([]string, error) +} + +// UnmaskingWorkflowUserDirectory 用户领域：工单列表/详情等场景将用户 UID 解析为展示名。 +// 由 DMS service 层适配 UserUsecase 实现。 +type UnmaskingWorkflowUserDirectory interface { + GetUserNamesByUIDs(ctx context.Context, uids []string) (map[string]string, error) +} + +// UnmaskingWorkflowDBServiceDirectory 数据源领域：将 DBService UID 解析为实例名称。 +// 由 DMS service 层适配 DBServiceUsecase 实现。 +type UnmaskingWorkflowDBServiceDirectory interface { + GetDBServiceNamesByUIDs(ctx context.Context, uids []string) (map[string]string, error) +} diff --git a/internal/data_masking/biz/unmasking_workflow_ce.go b/internal/data_masking/biz/unmasking_workflow_ce.go new file mode 100644 index 00000000..2ccb3e44 --- /dev/null +++ b/internal/data_masking/biz/unmasking_workflow_ce.go @@ -0,0 +1,84 @@ +//go:build !dms + +package biz + +import ( + "context" +) + +type UnmaskingWorkflowUsecase struct { +} + +// UnmaskingSQL 非 dms 构建占位；与 enterprise 中间件读取的字段对齐。 +type UnmaskingSQL struct { + UID string + SQLContent string +} + +// UnmaskingWorkflow 非 dms 构建下的占位类型；enterprise 会读 UID / ProjectUID 等（如 data_export_workflow_ee）。 +type UnmaskingWorkflow struct { + UID string + ProjectUID string + ApplicantUID string + UnmaskingSQLs []*UnmaskingSQL +} + +type CreateUnmaskingWorkflowArgs struct{} + +type UnmaskingDBConfig struct{} + +type ListUnmaskingWorkflowsOption struct{} + +func NewUnmaskingWorkflowUsecase(ctx context.Context) *UnmaskingWorkflowUsecase { + return &UnmaskingWorkflowUsecase{} +} + +func (u *UnmaskingWorkflowUsecase) CreateUnmaskingWorkflow(ctx context.Context, args *CreateUnmaskingWorkflowArgs) (string, error) { + return "", nil +} + +func (u *UnmaskingWorkflowUsecase) GetUnmaskingWorkflow(ctx context.Context, workflowUID string) (*UnmaskingWorkflow, error) { + return nil, nil +} + +func (u *UnmaskingWorkflowUsecase) ListUnmaskingWorkflows(ctx context.Context, opt *ListUnmaskingWorkflowsOption) ([]*UnmaskingWorkflow, error) { + return nil, nil +} + +func (u *UnmaskingWorkflowUsecase) ApproveUnmaskingWorkflow(ctx context.Context, workflowUID string) error { + return nil +} + +func (u *UnmaskingWorkflowUsecase) RejectUnmaskingWorkflow(ctx context.Context, workflowUID string) error { + return nil +} + +func (u *UnmaskingWorkflowUsecase) CancelUnmaskingWorkflow(ctx context.Context, workflowUID string) error { + return nil +} + +func (u *UnmaskingWorkflowUsecase) GetUnmaskingWorkflowAssignees(ctx context.Context, projectUID, datasourceUID, applicantUID string) ([]string, error) { + return nil, nil +} + +func (u *UnmaskingWorkflowUsecase) CheckOriginalDataAccess(ctx context.Context, projectUID, datasourceUID, userUID string, credential *UnmaskingOriginalDataCredentialArgs) (bool, *UnmaskingWorkflow, error) { + return false, nil, nil +} + +// MarkWorkflowUsage 完整实现仅在 dms 构建中提供；非 dms 占位。 +func (u *UnmaskingWorkflowUsecase) MarkWorkflowUsage(_ context.Context, _, _, _ string, _ UnmaskingAction) error { + return nil +} + +// GetUnmaskingWorkflowDetail 完整实现仅在 dms 构建中提供；非 dms 占位（运行时 enterprise 且未启用 dms 时 usecase 一般为 nil）。 +func (u *UnmaskingWorkflowUsecase) GetUnmaskingWorkflowDetail(_ context.Context, _, _, _ string) (*UnmaskingWorkflow, error) { + return &UnmaskingWorkflow{}, nil +} + +// AnalyzeLineageAndBuildMaskingSnapshot 完整实现仅在 dms 构建中提供；非 dms 构建占位以满足跨标签编译。 +func (u *UnmaskingWorkflowUsecase) AnalyzeLineageAndBuildMaskingSnapshot( + _ context.Context, + _, _, _, _ string, +) (*AnalyzeResult, []*ColumnMaskingConfig) { + return nil, nil +} From 6133c313c6a683ed904c7bc72ad18f327064abca Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 08:58:11 +0000 Subject: [PATCH 02/27] ce refactor: move Confidence type and constants to a new file for better organization Refactored the Confidence type and its associated constants by moving them from types_ee.go to a newly created types.go file. This change improves code organization and clarity, making it easier to manage and understand the confidence levels used in sensitive data detection. --- internal/data_masking/core/types.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 internal/data_masking/core/types.go diff --git a/internal/data_masking/core/types.go b/internal/data_masking/core/types.go new file mode 100644 index 00000000..8df3416d --- /dev/null +++ b/internal/data_masking/core/types.go @@ -0,0 +1,14 @@ +package core + +// Confidence 置信度 (例如: High, Medium, Low) +// swagger:enum Confidence +type Confidence string + +const ( + // 高：高度确信为敏感数据 + ConfidenceHigh Confidence = "High" + // 中：中等确信为敏感数据 + ConfidenceMedium Confidence = "Medium" + // 低：低确信为敏感数据 + ConfidenceLow Confidence = "Low" +) From 569d83dfa01cf08de6a087fdc7d51aa263ef743f Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 09:05:38 +0000 Subject: [PATCH 03/27] ce feat: implement DownloadOriginalDataExportWorkflow functionality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 数据导出集成查看原文工单功能 Added the DownloadOriginalDataExportWorkflow method to the DataExportWorkflowUsecase, enabling the download of unmasked data export workflows as a zip file. This includes validation checks for project UID, workflow status, and user permissions. Updated related service files to support this functionality, ensuring compatibility with both enterprise and community editions. Enhanced error handling and logging for better traceability. --- .../apiserver/service/data_mask_controller.go | 344 ++++++++++++++++-- internal/dms/biz/data_export_workflow.go | 6 +- internal/dms/biz/data_export_workflow_ce.go | 4 + internal/dms/service/data_export_workflow.go | 13 + .../dms/service/data_export_workflow_ce.go | 16 + internal/dms/service/service.go | 4 +- 6 files changed, 350 insertions(+), 37 deletions(-) create mode 100644 internal/dms/service/data_export_workflow_ce.go diff --git a/internal/apiserver/service/data_mask_controller.go b/internal/apiserver/service/data_mask_controller.go index 560b9ab8..65b0d167 100644 --- a/internal/apiserver/service/data_mask_controller.go +++ b/internal/apiserver/service/data_mask_controller.go @@ -1,8 +1,11 @@ package service import ( + "errors" + aV1 "github.com/actiontech/dms/api/dms/service/v1" apiError "github.com/actiontech/dms/internal/apiserver/pkg/error" + pkgConst "github.com/actiontech/dms/internal/dms/pkg/constant" "github.com/actiontech/dms/pkg/dms-common/api/jwt" "github.com/labstack/echo/v4" ) @@ -48,6 +51,35 @@ func (ctl *DMSController) ListMaskingTemplates(c echo.Context) error { return NewOkRespWithReply(c, reply) } +// swagger:route GET /v1/dms/projects/{project_uid}/db_services/{db_service_uid}/schemas/{schema_name}/tables/{table_name}/columns DBStructure ListTableColumns +// +// List table columns (internal API for lineage analysis). +// +// responses: +// 200: body:ListTableColumnsReply +// default: body:GenericResp +func (ctl *DMSController) ListTableColumns(c echo.Context) error { + // 内部接口，仅允许sys/admin用户访问 + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + if currentUserUid != pkgConst.UIDOfUserSys && currentUserUid != pkgConst.UIDOfUserAdmin { + return NewErrResp(c, errors.New("insufficient permission"), apiError.UnauthorizedErr) + } + + req := new(aV1.ListTableColumnsReq) + if err := bindAndValidateReq(c, req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + + reply, err := ctl.DMS.ListTableColumns(c.Request().Context(), req) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + return NewOkRespWithReply(c, reply) +} + // swagger:operation POST /v1/dms/projects/{project_uid}/masking/templates Masking AddMaskingTemplate // // 新增脱敏模板。 @@ -468,75 +500,317 @@ func (ctl *DMSController) GetTableColumnMaskingDetails(c echo.Context) error { return NewOkRespWithReply(c, reply) } -// swagger:route GET /v1/dms/projects/{project_uid}/masking/approval-requests/pending Masking ListPendingApprovalRequests +// swagger:operation POST /v1/dms/projects/{project_uid}/masking/unmasking-workflows Masking CreateUnmaskingWorkflow // -// 查询待审批申请列表。 +// Create unmasking workflow. // -// responses: -// 200: body:ListPendingApprovalRequestsReply -// default: body:GenericResp -func (ctl *DMSController) ListPendingApprovalRequests(c echo.Context) error { - req := &aV1.ListPendingApprovalRequestsReq{} +// --- +// parameters: +// - name: project_uid +// description: project id +// in: path +// required: true +// type: string +// - name: unmasking_workflow +// description: unmasking workflow info +// in: body +// required: true +// schema: +// "$ref": "#/definitions/CreateUnmaskingWorkflowReq" +// +// responses: +// +// '200': +// description: Create unmasking workflow successfully +// schema: +// "$ref": "#/definitions/CreateUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) CreateUnmaskingWorkflow(c echo.Context) error { + req := &aV1.CreateUnmaskingWorkflowReq{} if err := bindAndValidateReq(c, req); err != nil { return NewErrResp(c, err, apiError.BadRequestErr) } - return NewOkRespWithReply(c, &aV1.ListPendingApprovalRequestsReply{}) + + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + + reply, err := ctl.DMS.CreateUnmaskingWorkflow(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + + return NewOkRespWithReply(c, reply) } -// swagger:route GET /v1/dms/projects/{project_uid}/masking/approval-requests/{request_id} Masking GetPlaintextAccessRequestDetail +// swagger:operation GET /v1/dms/projects/{project_uid}/masking/unmasking-workflows Masking ListUnmaskingWorkflows // -// 获取明文访问申请详情。 +// List unmasking workflows. // -// responses: -// 200: body:GetPlaintextAccessRequestDetailReply -// default: body:GenericResp -func (ctl *DMSController) GetPlaintextAccessRequestDetail(c echo.Context) error { - req := &aV1.GetPlaintextAccessRequestDetailReq{} +// --- +// parameters: +// - name: project_uid +// description: project id +// in: path +// required: true +// type: string +// - name: page_size +// description: the maximum count of workflows to be returned +// in: query +// required: true +// type: integer +// format: uint32 +// - name: page_index +// description: the offset of workflows to be returned, default is 0 +// in: query +// required: false +// type: integer +// format: uint32 +// - name: filter_by_approval_status +// description: filter the approval status +// in: query +// required: false +// type: string +// enum: [pending, approved, rejected, cancelled] +// - name: filter_by_usage_status +// description: filter the usage status +// in: query +// required: false +// type: string +// enum: [unviewed, viewed] +// - name: filter_by_db_service_uid +// description: filter db_service id +// in: query +// required: false +// type: string +// +// responses: +// +// '200': +// description: List unmasking workflows successfully +// schema: +// "$ref": "#/definitions/ListUnmaskingWorkflowsReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) ListUnmaskingWorkflows(c echo.Context) error { + req := &aV1.ListUnmaskingWorkflowsReq{} if err := bindAndValidateReq(c, req); err != nil { return NewErrResp(c, err, apiError.BadRequestErr) } - return NewOkRespWithReply(c, &aV1.GetPlaintextAccessRequestDetailReply{}) + + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + + reply, err := ctl.DMS.ListUnmaskingWorkflows(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + + return NewOkRespWithReply(c, reply) } -// swagger:operation POST /v1/dms/projects/{project_uid}/masking/approval-requests/{request_id}/decisions Masking ProcessApprovalRequest +// swagger:operation GET /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id} Masking GetUnmaskingWorkflow // -// 处理审批申请。 +// Get unmasking workflow detail. // // --- // parameters: // - name: project_uid -// description: 项目 UID +// description: project id // in: path // required: true // type: string -// - name: request_id -// description: 审批申请 ID +// - name: workflow_id +// description: workflow id // in: path // required: true -// type: integer -// - name: action -// description: 处理动作信息 +// type: string +// +// responses: +// +// '200': +// description: Get unmasking workflow detail successfully +// schema: +// "$ref": "#/definitions/GetUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) GetUnmaskingWorkflow(c echo.Context) error { + req := &aV1.GetUnmaskingWorkflowReq{} + if err := bindAndValidateReq(c, req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + + reply, err := ctl.DMS.GetUnmaskingWorkflow(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + + return NewOkRespWithReply(c, reply) +} + +// swagger:operation POST /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/approve Masking ApproveUnmaskingWorkflow +// +// Approve unmasking workflow. +// +// --- +// parameters: +// - name: project_uid +// description: project id +// in: path +// required: true +// type: string +// - name: workflow_id +// description: workflow id +// in: path +// required: true +// type: string +// - name: approve_unmasking_workflow +// description: approve unmasking workflow info // in: body // required: true // schema: -// "$ref": "#/definitions/ProcessApprovalRequestReq" +// "$ref": "#/definitions/ApproveUnmaskingWorkflow" // // responses: // -// '200': -// description: 成功处理审批申请 -// schema: -// "$ref": "#/definitions/ProcessApprovalRequestReply" -// default: -// description: 通用错误响应 +// '200': +// description: Approve unmasking workflow successfully +// schema: +// "$ref": "#/definitions/ApproveUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) ApproveUnmaskingWorkflow(c echo.Context) error { + req := &aV1.ApproveUnmaskingWorkflowReq{} + if err := bindAndValidateReq(c, req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + + err = ctl.DMS.ApproveUnmaskingWorkflow(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + + return NewOkRespWithReply(c, &aV1.ApproveUnmaskingWorkflowReply{}) +} + +// swagger:operation POST /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/reject Masking RejectUnmaskingWorkflow +// +// Reject unmasking workflow. +// +// --- +// parameters: +// - name: project_uid +// description: project id +// in: path +// required: true +// type: string +// - name: workflow_id +// description: workflow id +// in: path +// required: true +// type: string +// - name: reject_unmasking_workflow +// description: reject unmasking workflow info +// in: body +// required: true // schema: -// "$ref": "#/definitions/GenericResp" -func (ctl *DMSController) ProcessApprovalRequest(c echo.Context) error { - req := &aV1.ProcessApprovalRequestReq{} +// "$ref": "#/definitions/RejectUnmaskingWorkflow" +// +// responses: +// +// '200': +// description: Reject unmasking workflow successfully +// schema: +// "$ref": "#/definitions/RejectUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) RejectUnmaskingWorkflow(c echo.Context) error { + req := &aV1.RejectUnmaskingWorkflowReq{} if err := bindAndValidateReq(c, req); err != nil { return NewErrResp(c, err, apiError.BadRequestErr) } - return NewOkRespWithReply(c, &aV1.ProcessApprovalRequestReply{}) + + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + + err = ctl.DMS.RejectUnmaskingWorkflow(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + + return NewOkRespWithReply(c, &aV1.RejectUnmaskingWorkflowReply{}) +} + +// swagger:operation POST /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/cancel Masking CancelUnmaskingWorkflow +// +// Cancel unmasking workflow. +// +// --- +// parameters: +// - name: project_uid +// description: project id +// in: path +// required: true +// type: string +// - name: workflow_id +// description: workflow id +// in: path +// required: true +// type: string +// +// responses: +// +// '200': +// description: Cancel unmasking workflow successfully +// schema: +// "$ref": "#/definitions/CancelUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) CancelUnmaskingWorkflow(c echo.Context) error { + req := &aV1.CancelUnmaskingWorkflowReq{} + if err := bindAndValidateReq(c, req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + + err = ctl.DMS.CancelUnmaskingWorkflow(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + + return NewOkRespWithReply(c, &aV1.CancelUnmaskingWorkflowReply{}) } // swagger:route GET /v1/dms/projects/{project_uid}/masking/rules/{rule_id} Masking GetMaskingRuleDetail diff --git a/internal/dms/biz/data_export_workflow.go b/internal/dms/biz/data_export_workflow.go index 148acaf2..a574bedb 100644 --- a/internal/dms/biz/data_export_workflow.go +++ b/internal/dms/biz/data_export_workflow.go @@ -8,6 +8,8 @@ import ( pkgConst "github.com/actiontech/dms/internal/dms/pkg/constant" dmsCommonV1 "github.com/actiontech/dms/pkg/dms-common/api/dms/v1" utilLog "github.com/actiontech/dms/pkg/dms-common/pkg/log" + + dataMaskingBiz "github.com/actiontech/dms/internal/data_masking/biz" ) var ErrDataExportWorkflowNameDuplicate = errors.New("data export workflow name duplicate") @@ -135,11 +137,12 @@ type DataExportWorkflowUsecase struct { userUsecase *UserUsecase systemVariableUsecase *SystemVariableUsecase maskingTaskRepo MaskingTaskRepo + unmaskingWorkflowUsecase *dataMaskingBiz.UnmaskingWorkflowUsecase log *utilLog.Helper reportHost string } -func NewDataExportWorkflowUsecase(logger utilLog.Logger, tx TransactionGenerator, repo WorkflowRepo, dataExportTaskRepo DataExportTaskRepo, dbServiceRepo DBServiceRepo, maskingConfigRepo DataExportMaskingConfigRepo, maskingRuleRepo DataExportMaskingRuleRepo, opPermissionVerifyUsecase *OpPermissionVerifyUsecase, projectUsecase *ProjectUsecase, proxyTargetRepo ProxyTargetRepo, clusterUseCase *ClusterUsecase, webhookUsecase *WebHookConfigurationUsecase, userUsecase *UserUsecase, systemVariableUsecase *SystemVariableUsecase, maskingTaskRepo MaskingTaskRepo, reportHost string) *DataExportWorkflowUsecase { +func NewDataExportWorkflowUsecase(logger utilLog.Logger, tx TransactionGenerator, repo WorkflowRepo, dataExportTaskRepo DataExportTaskRepo, dbServiceRepo DBServiceRepo, maskingConfigRepo DataExportMaskingConfigRepo, maskingRuleRepo DataExportMaskingRuleRepo, opPermissionVerifyUsecase *OpPermissionVerifyUsecase, projectUsecase *ProjectUsecase, proxyTargetRepo ProxyTargetRepo, clusterUseCase *ClusterUsecase, webhookUsecase *WebHookConfigurationUsecase, userUsecase *UserUsecase, systemVariableUsecase *SystemVariableUsecase, maskingTaskRepo MaskingTaskRepo, unmaskingWorkflowUsecase *dataMaskingBiz.UnmaskingWorkflowUsecase, reportHost string) *DataExportWorkflowUsecase { return &DataExportWorkflowUsecase{ tx: tx, repo: repo, @@ -155,6 +158,7 @@ func NewDataExportWorkflowUsecase(logger utilLog.Logger, tx TransactionGenerator userUsecase: userUsecase, systemVariableUsecase: systemVariableUsecase, maskingTaskRepo: maskingTaskRepo, + unmaskingWorkflowUsecase: unmaskingWorkflowUsecase, log: utilLog.NewHelper(logger, utilLog.WithMessageKey("biz.dataExportWorkflow")), reportHost: reportHost, } diff --git a/internal/dms/biz/data_export_workflow_ce.go b/internal/dms/biz/data_export_workflow_ce.go index 181b3954..f3116ab1 100644 --- a/internal/dms/biz/data_export_workflow_ce.go +++ b/internal/dms/biz/data_export_workflow_ce.go @@ -39,6 +39,10 @@ func (d *DataExportWorkflowUsecase) ExportDataExportWorkflow(ctx context.Context return errNotDataExportWorkflow } +func (d *DataExportWorkflowUsecase) DownloadOriginalDataExportWorkflow(ctx context.Context, projectUid, workflowUid, currentUserUid, unmaskingWorkflowUid string) (string, []byte, error) { + return "", nil, errNotDataExportWorkflow +} + func (d *DataExportWorkflowUsecase) DownloadDataExportTask(ctx context.Context, userId string, req *dmsV1.DownloadDataExportTaskReq) (bool, string, error) { return false, "", errNotDataExportTask } diff --git a/internal/dms/service/data_export_workflow.go b/internal/dms/service/data_export_workflow.go index e02b6a1e..f1a2e3eb 100644 --- a/internal/dms/service/data_export_workflow.go +++ b/internal/dms/service/data_export_workflow.go @@ -248,6 +248,8 @@ func (d *DMSService) GetDataExportWorkflow(ctx context.Context, req *dmsV1.GetDa data.WorkflowRecord.Steps = append(data.WorkflowRecord.Steps, step) } + d.fillGetDataExportUnmaskingWorkflowSummary(ctx, req.DataExportWorkflowUid, data) + return &dmsV1.GetDataExportWorkflowReply{ Data: data, }, nil @@ -337,6 +339,12 @@ func (d *DMSService) ListDataExportTaskSQLs(ctx context.Context, req *dmsV1.List return nil, err } + tasks, err := d.DataExportWorkflowUsecase.BatchGetDataExportTask(ctx, []string{req.DataExportTaskUid}) + if err != nil || len(tasks) == 0 { + return nil, fmt.Errorf("failed to get data export task: %w", err) + } + task := tasks[0] + ret := make([]*dmsV1.ListDataExportTaskSQL, len(taskRecords)) for i, w := range taskRecords { ret[i] = &dmsV1.ListDataExportTaskSQL{ @@ -346,6 +354,11 @@ func (d *DMSService) ListDataExportTaskSQLs(ctx context.Context, req *dmsV1.List ExportResult: w.ExportResult, ExportSQLType: w.ExportSQLType, } + if d.UnmaskingWorkflowUsecase != nil { + lineage, snapshot := d.UnmaskingWorkflowUsecase.AnalyzeLineageAndBuildMaskingSnapshot(ctx, req.ProjectUid, task.DBServiceUid, task.DatabaseName, w.ExportSQL) + ret[i].LineageAnalysisSnapshot = lineage + ret[i].MaskingConfigSnapshot = snapshot + } if w.AuditSQLResults != nil { for _, result := range w.AuditSQLResults { ret[i].AuditSQLResult = append(ret[i].AuditSQLResult, dmsV1.AuditSQLResult{ diff --git a/internal/dms/service/data_export_workflow_ce.go b/internal/dms/service/data_export_workflow_ce.go new file mode 100644 index 00000000..67de1d38 --- /dev/null +++ b/internal/dms/service/data_export_workflow_ce.go @@ -0,0 +1,16 @@ +//go:build !dms + +package service + +import ( + "context" + "errors" + + dmsV1 "github.com/actiontech/dms/api/dms/service/v1" +) + +func (d *DMSService) DownloadOriginalDataExportWorkflow(ctx context.Context, req *dmsV1.DownloadOriginalDataExportWorkflowReq, currentUserUid string) (string, []byte, error) { + return "", nil, errors.New("export original data is an enterprise version function") +} + +func (d *DMSService) fillGetDataExportUnmaskingWorkflowSummary(_ context.Context, _ string, _ *dmsV1.GetDataExportWorkflow) {} diff --git a/internal/dms/service/service.go b/internal/dms/service/service.go index 46c3fb0c..de9818cc 100644 --- a/internal/dms/service/service.go +++ b/internal/dms/service/service.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/actiontech/dms/internal/apiserver/conf" + maskingBiz "github.com/actiontech/dms/internal/data_masking/biz" "github.com/actiontech/dms/internal/dms/biz" "github.com/actiontech/dms/internal/dms/storage" @@ -43,6 +44,7 @@ type DMSService struct { DataExportWorkflowUsecase *biz.DataExportWorkflowUsecase CbOperationLogUsecase *biz.CbOperationLogUsecase DataMaskingUsecase *dataMaskingUsecase + UnmaskingWorkflowUsecase *maskingBiz.UnmaskingWorkflowUsecase FunctionSupportRegistry *biz.FunctionSupportRegistry AuthAccessTokenUseCase *biz.AuthAccessTokenUsecase SwaggerUseCase *biz.SwaggerUseCase @@ -153,7 +155,7 @@ func NewAndInitDMSService(logger utilLog.Logger, opts *conf.DMSOptions) (*DMSSer workflowRepo := storage.NewWorkflowRepo(logger, st) dataExportMaskingConfigRepo := initDataExportMaskingConfigRepo(logger, st) dataExportMaskingRuleRepo := initDataExportMaskingRuleRepo(logger, st) - DataExportWorkflowUsecase := biz.NewDataExportWorkflowUsecase(logger, tx, workflowRepo, dataExportTaskRepo, dbServiceRepo, dataExportMaskingConfigRepo, dataExportMaskingRuleRepo, opPermissionVerifyUsecase, projectUsecase, dmsProxyTargetRepo, clusterUsecase, webhookConfigurationUsecase, userUsecase, systemVariableUsecase, discoveryTaskRepo, fmt.Sprintf("%s:%d", opts.ReportHost, opts.APIServiceOpts.Port)) + DataExportWorkflowUsecase := biz.NewDataExportWorkflowUsecase(logger, tx, workflowRepo, dataExportTaskRepo, dbServiceRepo, dataExportMaskingConfigRepo, dataExportMaskingRuleRepo, opPermissionVerifyUsecase, projectUsecase, dmsProxyTargetRepo, clusterUsecase, webhookConfigurationUsecase, userUsecase, systemVariableUsecase, discoveryTaskRepo, nil, fmt.Sprintf("%s:%d", opts.ReportHost, opts.APIServiceOpts.Port)) dataMaskingUsecase, stopDataMaskingScheduler, err := initDataMaskingUsecase(logger, st, dbServiceUseCase, clusterUsecase, dmsProxyTargetRepo) if err != nil { return nil, fmt.Errorf("failed to initialize data masking usecase: %v", err) From c625fd29fe2b7879083025fb0de88cd1a7b32e9f Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Mon, 11 May 2026 12:08:44 +0000 Subject: [PATCH 04/27] ce feat: DMSService: add UnmaskingWorkflowUsecase field for export original (CE) Owns the service struct wiring previously marked ee; pairs with biz/data_export EE implementation. --- internal/dms/service/service.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/dms/service/service.go b/internal/dms/service/service.go index de9818cc..10cdeeac 100644 --- a/internal/dms/service/service.go +++ b/internal/dms/service/service.go @@ -42,9 +42,9 @@ type DMSService struct { LicenseUsecase *biz.LicenseUsecase ClusterUsecase *biz.ClusterUsecase DataExportWorkflowUsecase *biz.DataExportWorkflowUsecase + UnmaskingWorkflowUsecase *maskingBiz.UnmaskingWorkflowUsecase CbOperationLogUsecase *biz.CbOperationLogUsecase DataMaskingUsecase *dataMaskingUsecase - UnmaskingWorkflowUsecase *maskingBiz.UnmaskingWorkflowUsecase FunctionSupportRegistry *biz.FunctionSupportRegistry AuthAccessTokenUseCase *biz.AuthAccessTokenUsecase SwaggerUseCase *biz.SwaggerUseCase @@ -204,6 +204,7 @@ func NewAndInitDMSService(logger utilLog.Logger, opts *conf.DMSOptions) (*DMSSer LicenseUsecase: LicenseUsecase, ClusterUsecase: clusterUsecase, DataExportWorkflowUsecase: DataExportWorkflowUsecase, + UnmaskingWorkflowUsecase: nil, CbOperationLogUsecase: CbOperationLogUsecase, DataMaskingUsecase: dataMaskingUsecase, FunctionSupportRegistry: functionSupportRegistry, From 3273512e1fb6e6bf22c75691a9f3eefcb7303e4f Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 09:07:54 +0000 Subject: [PATCH 05/27] ce feat: enhance SQL result masking with project UID support Updated the SQLResultMasker interface to include projectUID as a parameter for masking SQL results. Modified the CloudbeaverUsecase to pass projectUID during data masking operations. Introduced a new SQLResultMasker implementation for SQL workbench results, allowing for improved context and handling of masking operations. Refactored response writer to ensure consistent handling of response statuses and headers. --- internal/dms/biz/cloudbeaver.go | 9 ++- .../service/sql_workbench_service.go | 62 +++++++++++-------- .../sql_workbench/sqlresultmasker/masker.go | 18 ++++++ 3 files changed, 60 insertions(+), 29 deletions(-) create mode 100644 internal/sql_workbench/sqlresultmasker/masker.go diff --git a/internal/dms/biz/cloudbeaver.go b/internal/dms/biz/cloudbeaver.go index b0b1e832..25eb5e11 100644 --- a/internal/dms/biz/cloudbeaver.go +++ b/internal/dms/biz/cloudbeaver.go @@ -65,7 +65,7 @@ func (c CloudbeaverConnection) PrimaryKey() string { } type SQLResultMasker interface { - MaskSQLResults(ctx context.Context, result *model.SQLExecuteInfo, dbServiceUID, schemaName string) error + MaskSQLResults(ctx context.Context, result *model.SQLExecuteInfo, dbServiceUID, schemaName, projectUID string) error } type CloudbeaverRepo interface { @@ -339,6 +339,7 @@ type taskMaskingContext struct { Enabled bool DBServiceUID string SchemaName string + ProjectUID string } var ( @@ -471,6 +472,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { return cu.buildTaskIdAssocDataMasking(cloudbeaverResBuf.Bytes(), taskMaskingContext{ Enabled: isMaskingEnabled, DBServiceUID: dbService.UID, + ProjectUID: dbService.ProjectUID, }) } @@ -499,6 +501,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { maskCtx := taskMaskingContext{ Enabled: isMaskingEnabled, DBServiceUID: dbService.UID, + ProjectUID: dbService.ProjectUID, } if ep, epErr := cu.getWorkflowExecParams(c, params); epErr == nil { maskCtx.SchemaName = ep.instanceSchema @@ -569,6 +572,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { return cu.buildTaskIdAssocDataMasking(cloudbeaverResBuf.Bytes(), taskMaskingContext{ Enabled: isMaskingEnabled, DBServiceUID: dbService.UID, + ProjectUID: dbService.ProjectUID, }) } @@ -703,6 +707,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { Enabled: isMaskingEnabled, DBServiceUID: dbService.UID, SchemaName: maskingSchemaName, + ProjectUID: dbService.ProjectUID, }); err != nil { return nil, err } @@ -754,7 +759,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { if cu.sqlResultMasker == nil { return nil } - return cu.sqlResultMasker.MaskSQLResults(ctx, result, maskingCtx.DBServiceUID, maskingCtx.SchemaName) + return cu.sqlResultMasker.MaskSQLResults(ctx, result, maskingCtx.DBServiceUID, maskingCtx.SchemaName, maskingCtx.ProjectUID) } // 创建GraphQL可执行schema diff --git a/internal/sql_workbench/service/sql_workbench_service.go b/internal/sql_workbench/service/sql_workbench_service.go index e2ffff1b..9e3d4913 100644 --- a/internal/sql_workbench/service/sql_workbench_service.go +++ b/internal/sql_workbench/service/sql_workbench_service.go @@ -23,10 +23,11 @@ import ( "github.com/actiontech/dms/internal/dms/biz" pkgConst "github.com/actiontech/dms/internal/dms/pkg/constant" "github.com/actiontech/dms/internal/dms/storage" - "github.com/actiontech/dms/internal/pkg/locale" dbmodel "github.com/actiontech/dms/internal/dms/storage/model" + "github.com/actiontech/dms/internal/pkg/locale" "github.com/actiontech/dms/internal/sql_workbench/client" config "github.com/actiontech/dms/internal/sql_workbench/config" + "github.com/actiontech/dms/internal/sql_workbench/sqlresultmasker" "github.com/actiontech/dms/pkg/dms-common/api/jwt" "github.com/actiontech/dms/pkg/dms-common/i18nPkg" _const "github.com/actiontech/dms/pkg/dms-common/pkg/const" @@ -102,6 +103,15 @@ type SqlWorkbenchService struct { sqlWorkbenchDatasourceRepo biz.SqlWorkbenchDatasourceRepo proxyTargetRepo biz.ProxyTargetRepo cbOperationLogUsecase *biz.CbOperationLogUsecase + sqlResultMasker sqlresultmasker.SQLResultMasker +} + +func (sqlWorkbenchService *SqlWorkbenchService) SetSqlResultMasker(masker sqlresultmasker.SQLResultMasker) { + sqlWorkbenchService.sqlResultMasker = masker +} + +func (sqlWorkbenchService *SqlWorkbenchService) GetSqlResultMasker() sqlresultmasker.SQLResultMasker { + return sqlWorkbenchService.sqlResultMasker } func NewAndInitSqlWorkbenchService(logger utilLog.Logger, opts *conf.DMSOptions) (*SqlWorkbenchService, error) { @@ -1438,14 +1448,14 @@ func extractSessionID(path string) string { // executeAndAddAuditResult 执行真实请求并添加审核结果 func (sqlWorkbenchService *SqlWorkbenchService) executeAndAddAuditResult(c echo.Context, next echo.HandlerFunc, auditResult *cloudbeaver.AuditSQLReply, dbService *biz.DBService) error { // 创建响应拦截器 - srw := newStreamExecuteResponseWriter(c) + srw := NewStreamExecuteResponseWriter(c) cloudbeaverResBuf := srw.Buffer c.Response().Writer = srw defer func() { // 在 defer 中处理响应 - if srw.status != 0 { - srw.original.WriteHeader(srw.status) + if srw.Status != 0 { + srw.Original.WriteHeader(srw.Status) } // 读取响应内容 @@ -1458,13 +1468,13 @@ func (sqlWorkbenchService *SqlWorkbenchService) executeAndAddAuditResult(c echo. responseBytes, wasGzip, err := sqlWorkbenchService.decodeResponseBody(cloudbeaverResBuf.Bytes(), srw.Header().Get("Content-Encoding")) if err != nil { sqlWorkbenchService.log.Debugf("Failed to decode response body, returning original response: %v", err) - srw.original.Write(cloudbeaverResBuf.Bytes()) + srw.Original.Write(cloudbeaverResBuf.Bytes()) return } // 如果解压过，先移除 Content-Encoding，后续根据需要重新设置 if wasGzip { - srw.original.Header().Del("Content-Encoding") + srw.Original.Header().Del("Content-Encoding") } // 解析响应 JSON @@ -1472,7 +1482,7 @@ func (sqlWorkbenchService *SqlWorkbenchService) executeAndAddAuditResult(c echo. if err := json.Unmarshal(responseBytes, &responseBody); err != nil { sqlWorkbenchService.log.Debugf("Failed to unmarshal response, returning original response: %v", err) // 如果解析失败，直接返回原始响应 - srw.original.Write(cloudbeaverResBuf.Bytes()) + srw.Original.Write(cloudbeaverResBuf.Bytes()) return } @@ -1486,7 +1496,7 @@ func (sqlWorkbenchService *SqlWorkbenchService) executeAndAddAuditResult(c echo. modifiedResponse, err := json.Marshal(responseBody) if err != nil { sqlWorkbenchService.log.Errorf("Failed to marshal modified response: %v", err) - srw.original.Write(cloudbeaverResBuf.Bytes()) + srw.Original.Write(cloudbeaverResBuf.Bytes()) return } @@ -1495,26 +1505,26 @@ func (sqlWorkbenchService *SqlWorkbenchService) executeAndAddAuditResult(c echo. encoded, err := sqlWorkbenchService.encodeResponseBody(modifiedResponse) if err != nil { sqlWorkbenchService.log.Errorf("Failed to re-encode gzip response: %v", err) - srw.original.Write(cloudbeaverResBuf.Bytes()) + srw.Original.Write(cloudbeaverResBuf.Bytes()) return } finalResponse = encoded - srw.original.Header().Set("Content-Encoding", "gzip") + srw.Original.Header().Set("Content-Encoding", "gzip") } // 更新 Content-Length - header := srw.original.Header() + header := srw.Original.Header() header.Set("Content-Length", fmt.Sprintf("%d", len(finalResponse))) // 如果拦截过程中未显式写入状态码，默认使用 200 - statusCode := srw.status + statusCode := srw.Status if statusCode == 0 { statusCode = http.StatusOK } - srw.original.WriteHeader(statusCode) + srw.Original.WriteHeader(statusCode) // 写入修改后的响应 - if _, err := srw.original.Write(finalResponse); err != nil { + if _, err := srw.Original.Write(finalResponse); err != nil { sqlWorkbenchService.log.Errorf("Failed to write modified response: %v", err) } }() @@ -1747,34 +1757,32 @@ type SQLEAuditResultSummary struct { PassRate float64 `json:"pass_rate"` } -// streamExecuteResponseWriter 响应拦截器，用于捕获响应内容 -type streamExecuteResponseWriter struct { +// StreamExecuteResponseWriter 响应拦截器，用于捕获响应内容 +type StreamExecuteResponseWriter struct { echo.Response Buffer *bytes.Buffer - original http.ResponseWriter - status int + Original http.ResponseWriter + Status int } -func newStreamExecuteResponseWriter(c echo.Context) *streamExecuteResponseWriter { +func NewStreamExecuteResponseWriter(c echo.Context) *StreamExecuteResponseWriter { buf := new(bytes.Buffer) - return &streamExecuteResponseWriter{ + return &StreamExecuteResponseWriter{ Response: *c.Response(), Buffer: buf, - original: c.Response().Writer, + Original: c.Response().Writer, } } -func (w *streamExecuteResponseWriter) Write(b []byte) (int, error) { +func (w *StreamExecuteResponseWriter) Write(b []byte) (int, error) { // 如果未设置状态码，则补默认值 - if w.status == 0 { + if w.Status == 0 { w.WriteHeader(http.StatusOK) } // 写入 buffer，不立即写给客户端 return w.Buffer.Write(b) } -func (w *streamExecuteResponseWriter) WriteHeader(code int) { - w.status = code +func (w *StreamExecuteResponseWriter) WriteHeader(code int) { + w.Status = code } - - diff --git a/internal/sql_workbench/sqlresultmasker/masker.go b/internal/sql_workbench/sqlresultmasker/masker.go new file mode 100644 index 00000000..780c02f3 --- /dev/null +++ b/internal/sql_workbench/sqlresultmasker/masker.go @@ -0,0 +1,18 @@ +package sqlresultmasker + +import "context" + +// MaskWorkbenchResultsArgs carries tabular SQL workbench (ODC) result data and masking scope. Rows are masked in place. +type MaskWorkbenchResultsArgs struct { + Rows [][]any `json:"rows"` + ColumnNames []string `json:"column_names"` + SQL string `json:"sql"` + DBServiceUID string `json:"db_service_uid"` + SchemaName string `json:"schema_name"` + ProjectUID string `json:"project_uid"` +} + +// SQLResultMasker masks SQL workbench (ODC) tabular result rows in place and reports which columns were masked. +type SQLResultMasker interface { + MaskSQLWorkbenchResults(ctx context.Context, args *MaskWorkbenchResultsArgs) (map[string]bool, error) +} From f9920ed5504872cb068a5e45e107c3b6768008fb Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 09:09:36 +0000 Subject: [PATCH 06/27] ce feat: implement unmasking workflow endpoints and middleware Added new endpoints for managing unmasking workflows, including creation, retrieval, approval, rejection, and cancellation. Introduced middleware for handling data masking and unmasking operations in SQL Workbench, ensuring proper permission checks and response handling. Enhanced the API structure to support these functionalities, improving overall data management capabilities. --- .../service/data_masking_middleware.go | 15 ++++++++++++ .../service/data_masking_middleware_ce.go | 24 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 internal/sql_workbench/service/data_masking_middleware.go create mode 100644 internal/sql_workbench/service/data_masking_middleware_ce.go diff --git a/internal/sql_workbench/service/data_masking_middleware.go b/internal/sql_workbench/service/data_masking_middleware.go new file mode 100644 index 00000000..46e0dc76 --- /dev/null +++ b/internal/sql_workbench/service/data_masking_middleware.go @@ -0,0 +1,15 @@ +package sql_workbench + +import ( + "github.com/actiontech/dms/internal/dms/biz" + dataMaskingBiz "github.com/actiontech/dms/internal/data_masking/biz" + "github.com/actiontech/dms/internal/sql_workbench/sqlresultmasker" +) + +// DataMaskingMiddlewareConfig 配置脱敏中间件 +type DataMaskingMiddlewareConfig struct { + SqlResultMasker sqlresultmasker.SQLResultMasker + DBServiceUsecase *biz.DBServiceUsecase + SqlWorkbenchService *SqlWorkbenchService + UnmaskingWorkflowUsecase *dataMaskingBiz.UnmaskingWorkflowUsecase +} diff --git a/internal/sql_workbench/service/data_masking_middleware_ce.go b/internal/sql_workbench/service/data_masking_middleware_ce.go new file mode 100644 index 00000000..5f10b904 --- /dev/null +++ b/internal/sql_workbench/service/data_masking_middleware_ce.go @@ -0,0 +1,24 @@ +//go:build !enterprise +// +build !enterprise + +package sql_workbench + +import "github.com/labstack/echo/v4" + +// GetDataMaskingMiddleware 返回用于脱敏的中间件 +func GetDataMaskingMiddleware(config DataMaskingMiddlewareConfig) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + return next(c) + } + } +} + +// GetUnmaskingWorkflowMiddleware 处理获批后查看原文，执行 SQL 之前进行替换 +func GetUnmaskingWorkflowMiddleware(config DataMaskingMiddlewareConfig) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + return next(c) + } + } +} From c0b3a694e44d4faa4040d1e5bb885d45ee86f652 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Mon, 11 May 2026 12:08:44 +0000 Subject: [PATCH 07/27] ce feat: enhance DataExportWorkflowUsecase with projectUID and unmasking workflow functionality Updated the newExportMaskingTransfer method to include projectUID as a parameter, improving the masking transfer capabilities. Added markUnmaskingWorkflowUsage method to track unmasking workflow usage. Initialized unmaskingWorkflowUsecase in DMSService, enabling unmasking workflow management in the enterprise edition. Introduced new SQL workbench result masker implementations for both community and enterprise editions, ensuring proper handling based on build tags. --- internal/dms/service/service.go | 8 +++- .../service/sql_workbench_result_masker_ce.go | 14 ++++++ internal/dms/service/unmasking_workflow_ce.go | 46 +++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 internal/dms/service/sql_workbench_result_masker_ce.go create mode 100644 internal/dms/service/unmasking_workflow_ce.go diff --git a/internal/dms/service/service.go b/internal/dms/service/service.go index 10cdeeac..3763b056 100644 --- a/internal/dms/service/service.go +++ b/internal/dms/service/service.go @@ -155,7 +155,11 @@ func NewAndInitDMSService(logger utilLog.Logger, opts *conf.DMSOptions) (*DMSSer workflowRepo := storage.NewWorkflowRepo(logger, st) dataExportMaskingConfigRepo := initDataExportMaskingConfigRepo(logger, st) dataExportMaskingRuleRepo := initDataExportMaskingRuleRepo(logger, st) - DataExportWorkflowUsecase := biz.NewDataExportWorkflowUsecase(logger, tx, workflowRepo, dataExportTaskRepo, dbServiceRepo, dataExportMaskingConfigRepo, dataExportMaskingRuleRepo, opPermissionVerifyUsecase, projectUsecase, dmsProxyTargetRepo, clusterUsecase, webhookConfigurationUsecase, userUsecase, systemVariableUsecase, discoveryTaskRepo, nil, fmt.Sprintf("%s:%d", opts.ReportHost, opts.APIServiceOpts.Port)) + unmaskingWorkflowUsecase, err := initUnmaskingWorkflowUsecase(logger, st, dmsProxyTargetRepo, opPermissionVerifyUsecase, userUsecase, dbServiceUseCase) + if err != nil { + return nil, fmt.Errorf("failed to initialize unmasking workflow usecase: %v", err) + } + DataExportWorkflowUsecase := biz.NewDataExportWorkflowUsecase(logger, tx, workflowRepo, dataExportTaskRepo, dbServiceRepo, dataExportMaskingConfigRepo, dataExportMaskingRuleRepo, opPermissionVerifyUsecase, projectUsecase, dmsProxyTargetRepo, clusterUsecase, webhookConfigurationUsecase, userUsecase, systemVariableUsecase, discoveryTaskRepo, unmaskingWorkflowUsecase, fmt.Sprintf("%s:%d", opts.ReportHost, opts.APIServiceOpts.Port)) dataMaskingUsecase, stopDataMaskingScheduler, err := initDataMaskingUsecase(logger, st, dbServiceUseCase, clusterUsecase, dmsProxyTargetRepo) if err != nil { return nil, fmt.Errorf("failed to initialize data masking usecase: %v", err) @@ -204,7 +208,7 @@ func NewAndInitDMSService(logger utilLog.Logger, opts *conf.DMSOptions) (*DMSSer LicenseUsecase: LicenseUsecase, ClusterUsecase: clusterUsecase, DataExportWorkflowUsecase: DataExportWorkflowUsecase, - UnmaskingWorkflowUsecase: nil, + UnmaskingWorkflowUsecase: unmaskingWorkflowUsecase, CbOperationLogUsecase: CbOperationLogUsecase, DataMaskingUsecase: dataMaskingUsecase, FunctionSupportRegistry: functionSupportRegistry, diff --git a/internal/dms/service/sql_workbench_result_masker_ce.go b/internal/dms/service/sql_workbench_result_masker_ce.go new file mode 100644 index 00000000..de721927 --- /dev/null +++ b/internal/dms/service/sql_workbench_result_masker_ce.go @@ -0,0 +1,14 @@ +//go:build !dms + +package service + +import ( + "github.com/actiontech/dms/internal/dms/storage" + sqlresultmasker "github.com/actiontech/dms/internal/sql_workbench/sqlresultmasker" + utilLog "github.com/actiontech/dms/pkg/dms-common/pkg/log" +) + +// NewSQLWorkbenchSQLResultMasker is a no-op in builds without the dms data-masking stack. +func NewSQLWorkbenchSQLResultMasker(_ utilLog.Logger, _ *storage.Storage) (sqlresultmasker.SQLResultMasker, error) { + return nil, nil +} diff --git a/internal/dms/service/unmasking_workflow_ce.go b/internal/dms/service/unmasking_workflow_ce.go new file mode 100644 index 00000000..4cf7bd29 --- /dev/null +++ b/internal/dms/service/unmasking_workflow_ce.go @@ -0,0 +1,46 @@ +//go:build !dms + +package service + +import ( + "context" + "errors" + + v1 "github.com/actiontech/dms/api/dms/service/v1" + "github.com/actiontech/dms/internal/data_masking/biz" + dmsBiz "github.com/actiontech/dms/internal/dms/biz" + "github.com/actiontech/dms/internal/dms/storage" + utilLog "github.com/actiontech/dms/pkg/dms-common/pkg/log" +) + +var errNotSupportUnmaskingWorkflow = errors.New("unmasking workflow related functions are enterprise version functions") + +type unmaskingWorkflowUsecase = biz.UnmaskingWorkflowUsecase + +func initUnmaskingWorkflowUsecase(_ utilLog.Logger, _ *storage.Storage, _ dmsBiz.ProxyTargetRepo, _ *dmsBiz.OpPermissionVerifyUsecase, _ *dmsBiz.UserUsecase, _ *dmsBiz.DBServiceUsecase) (*unmaskingWorkflowUsecase, error) { + return nil, nil +} + +func (d *DMSService) CreateUnmaskingWorkflow(ctx context.Context, req *v1.CreateUnmaskingWorkflowReq, currentUserUid string) (*v1.CreateUnmaskingWorkflowReply, error) { + return nil, errNotSupportUnmaskingWorkflow +} + +func (d *DMSService) GetUnmaskingWorkflow(ctx context.Context, req *v1.GetUnmaskingWorkflowReq, currentUserUid string) (*v1.GetUnmaskingWorkflowReply, error) { + return nil, errNotSupportUnmaskingWorkflow +} + +func (d *DMSService) ListUnmaskingWorkflows(ctx context.Context, req *v1.ListUnmaskingWorkflowsReq, currentUserUid string) (*v1.ListUnmaskingWorkflowsReply, error) { + return nil, errNotSupportUnmaskingWorkflow +} + +func (d *DMSService) ApproveUnmaskingWorkflow(ctx context.Context, req *v1.ApproveUnmaskingWorkflowReq, currentUserUid string) error { + return errNotSupportUnmaskingWorkflow +} + +func (d *DMSService) RejectUnmaskingWorkflow(ctx context.Context, req *v1.RejectUnmaskingWorkflowReq, currentUserUid string) error { + return errNotSupportUnmaskingWorkflow +} + +func (d *DMSService) CancelUnmaskingWorkflow(ctx context.Context, req *v1.CancelUnmaskingWorkflowReq, currentUserUid string) error { + return errNotSupportUnmaskingWorkflow +} From 5f98f558b8ce29dccc3f2f2dc928e3c798f27596 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Mon, 11 May 2026 12:08:45 +0000 Subject: [PATCH 08/27] ce feat: DMSService: align UnmaskingWorkflowUsecase field type (CE) Owns the service struct type alignment previously marked ee; pairs with storage/model EE implementation. --- internal/dms/service/service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/dms/service/service.go b/internal/dms/service/service.go index 3763b056..d69cb01c 100644 --- a/internal/dms/service/service.go +++ b/internal/dms/service/service.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/actiontech/dms/internal/apiserver/conf" - maskingBiz "github.com/actiontech/dms/internal/data_masking/biz" "github.com/actiontech/dms/internal/dms/biz" "github.com/actiontech/dms/internal/dms/storage" @@ -42,7 +41,7 @@ type DMSService struct { LicenseUsecase *biz.LicenseUsecase ClusterUsecase *biz.ClusterUsecase DataExportWorkflowUsecase *biz.DataExportWorkflowUsecase - UnmaskingWorkflowUsecase *maskingBiz.UnmaskingWorkflowUsecase + UnmaskingWorkflowUsecase *unmaskingWorkflowUsecase CbOperationLogUsecase *biz.CbOperationLogUsecase DataMaskingUsecase *dataMaskingUsecase FunctionSupportRegistry *biz.FunctionSupportRegistry From 337781de93fd5d39032f96064fd774dcca45abdf Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 09:14:57 +0000 Subject: [PATCH 09/27] ce feat: add ListTableColumns method for DMSService Introduced the ListTableColumns method in the DMSService to handle requests for listing table columns. This method currently returns an error indicating unsupported data masking, laying the groundwork for future enhancements in data management capabilities. --- internal/dms/service/db_structures_ce.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 internal/dms/service/db_structures_ce.go diff --git a/internal/dms/service/db_structures_ce.go b/internal/dms/service/db_structures_ce.go new file mode 100644 index 00000000..de768ef0 --- /dev/null +++ b/internal/dms/service/db_structures_ce.go @@ -0,0 +1,14 @@ +//go:build !dms + +package service + +import ( + "context" + + dmsV1 "github.com/actiontech/dms/api/dms/service/v1" +) + +func (d *DMSService) ListTableColumns(ctx context.Context, req *dmsV1.ListTableColumnsReq) (*dmsV1.ListTableColumnsReply, error) { + return nil, errNotSupportDataMasking +} + From 793644cc52a4468f4ac1a0a98b16b2a05e58023f Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 09:15:29 +0000 Subject: [PATCH 10/27] ce feat: add unmasking workflow messages for English and Chinese locales Updated the locale files to include new messages related to unmasking workflows, enhancing user feedback for operations such as submission, approval, cancellation, and viewing details. This addition supports the recently implemented unmasking workflow features in the DMS service. --- internal/pkg/locale/active.en.toml | 9 +++ internal/pkg/locale/active.zh.toml | 9 +++ internal/pkg/locale/message_zh.go | 107 ++++++++++++++++------------- 3 files changed, 79 insertions(+), 46 deletions(-) diff --git a/internal/pkg/locale/active.en.toml b/internal/pkg/locale/active.en.toml index 9e362dfc..06e8c616 100644 --- a/internal/pkg/locale/active.en.toml +++ b/internal/pkg/locale/active.en.toml @@ -179,6 +179,15 @@ OpRecordDataExportCreate = "Create data export workflow" OpRecordDataExportCreateWithName = "Create data export workflow %s" OpRecordDataExportExportWithName = "Execute data export %s" OpRecordDataExportRejectWithName = "Reject data export workflow %s" +OpRecordUnmaskingAPICreate = "Submit unmasking workflow application" +OpRecordUnmaskingApproveWithWorkflowUID = "Approve unmasking workflow %s" +OpRecordUnmaskingCancelWithWorkflowUID = "Cancel unmasking workflow %s" +OpRecordUnmaskingDownloadOriginalWithWorkflowUID = "Download original data %s" +OpRecordUnmaskingRejectWithWorkflowUID = "Reject unmasking workflow %s" +OpRecordUnmaskingSubmitWithWorkflowUID = "Submit unmasking workflow %s" +OpRecordUnmaskingUnknownWithWorkflowUID = "Unmasking workflow action %s (%s)" +OpRecordUnmaskingViewDetailWithWorkflowUID = "View unmasking workflow detail %s" +OpRecordUnmaskingViewOriginalWithWorkflowUID = "View full original data (SQL) %s" OpRecordMemberCreate = "Add member" OpRecordMemberCreateWithName = "Add member %s" OpRecordMemberDelete = "Delete member %s" diff --git a/internal/pkg/locale/active.zh.toml b/internal/pkg/locale/active.zh.toml index 2186794c..80b05e6a 100644 --- a/internal/pkg/locale/active.zh.toml +++ b/internal/pkg/locale/active.zh.toml @@ -179,6 +179,15 @@ OpRecordDataExportCreate = "创建数据导出工单" OpRecordDataExportCreateWithName = "创建数据导出工单 %s" OpRecordDataExportExportWithName = "执行数据导出 %s" OpRecordDataExportRejectWithName = "驳回数据导出工单 %s" +OpRecordUnmaskingAPICreate = "提交查看原文工单申请" +OpRecordUnmaskingApproveWithWorkflowUID = "审批通过查看原文工单 %s" +OpRecordUnmaskingCancelWithWorkflowUID = "撤回查看原文工单 %s" +OpRecordUnmaskingDownloadOriginalWithWorkflowUID = "下载原文数据 %s" +OpRecordUnmaskingRejectWithWorkflowUID = "驳回查看原文工单 %s" +OpRecordUnmaskingSubmitWithWorkflowUID = "提交查看原文工单 %s" +OpRecordUnmaskingUnknownWithWorkflowUID = "查看原文工单操作 %s (%s)" +OpRecordUnmaskingViewDetailWithWorkflowUID = "查看查看原文工单详情 %s" +OpRecordUnmaskingViewOriginalWithWorkflowUID = "查看原文（SQL）%s" OpRecordMemberCreate = "添加成员" OpRecordMemberCreateWithName = "添加成员 %s" OpRecordMemberDelete = "删除成员 %s" diff --git a/internal/pkg/locale/message_zh.go b/internal/pkg/locale/message_zh.go index 2ec18292..e0e6a66d 100644 --- a/internal/pkg/locale/message_zh.go +++ b/internal/pkg/locale/message_zh.go @@ -267,50 +267,65 @@ var ( // Operation Record var ( - OpRecordUserCreate = &i18n.Message{ID: "OpRecordUserCreate", Other: "创建用户"} - OpRecordUserCreateWithName = &i18n.Message{ID: "OpRecordUserCreateWithName", Other: "创建用户 %s"} - OpRecordCurrentUserUpdate = &i18n.Message{ID: "OpRecordCurrentUserUpdate", Other: "更新个人中心账号基本信息"} - OpRecordUserUpdate = &i18n.Message{ID: "OpRecordUserUpdate", Other: "更新用户 %s"} - OpRecordUserDelete = &i18n.Message{ID: "OpRecordUserDelete", Other: "删除用户 %s"} - OpRecordMemberCreate = &i18n.Message{ID: "OpRecordMemberCreate", Other: "添加成员"} - OpRecordMemberCreateWithName = &i18n.Message{ID: "OpRecordMemberCreateWithName", Other: "添加成员 %s"} - OpRecordMemberUpdate = &i18n.Message{ID: "OpRecordMemberUpdate", Other: "更新成员 %s"} - OpRecordMemberDelete = &i18n.Message{ID: "OpRecordMemberDelete", Other: "删除成员 %s"} - OpRecordMemberGroupCreate = &i18n.Message{ID: "OpRecordMemberGroupCreate", Other: "添加成员组"} - OpRecordMemberGroupCreateWithName = &i18n.Message{ID: "OpRecordMemberGroupCreateWithName", Other: "添加成员组 %s"} - OpRecordMemberGroupUpdate = &i18n.Message{ID: "OpRecordMemberGroupUpdate", Other: "更新成员组 %s"} - OpRecordMemberGroupDelete = &i18n.Message{ID: "OpRecordMemberGroupDelete", Other: "删除成员组 %s"} - OpRecordRoleCreate = &i18n.Message{ID: "OpRecordRoleCreate", Other: "创建角色"} - OpRecordRoleCreateWithName = &i18n.Message{ID: "OpRecordRoleCreateWithName", Other: "创建角色 %s"} - OpRecordRoleUpdate = &i18n.Message{ID: "OpRecordRoleUpdate", Other: "更新角色 %s"} - OpRecordRoleDelete = &i18n.Message{ID: "OpRecordRoleDelete", Other: "删除角色 %s"} - OpRecordProjectCreate = &i18n.Message{ID: "OpRecordProjectCreate", Other: "创建项目"} - OpRecordProjectCreateWithName = &i18n.Message{ID: "OpRecordProjectCreateWithName", Other: "创建项目 %s"} - OpRecordProjectUpdate = &i18n.Message{ID: "OpRecordProjectUpdate", Other: "更新项目 %s"} - OpRecordProjectDelete = &i18n.Message{ID: "OpRecordProjectDelete", Other: "删除项目 %s"} - OpRecordProjectArchive = &i18n.Message{ID: "OpRecordProjectArchive", Other: "归档项目 %s"} - OpRecordProjectUnarchive = &i18n.Message{ID: "OpRecordProjectUnarchive", Other: "取消归档项目 %s"} - OpRecordDBServiceCreate = &i18n.Message{ID: "OpRecordDBServiceCreate", Other: "创建数据源"} - OpRecordDBServiceCreateWithName = &i18n.Message{ID: "OpRecordDBServiceCreateWithName", Other: "创建数据源 %s"} - OpRecordDBServiceUpdate = &i18n.Message{ID: "OpRecordDBServiceUpdate", Other: "更新数据源 %s"} - OpRecordDBServiceDelete = &i18n.Message{ID: "OpRecordDBServiceDelete", Other: "删除数据源 %s"} - OpRecordDBServiceImport = &i18n.Message{ID: "OpRecordDBServiceImport", Other: "导入数据源"} - OpRecordConfigLogin = &i18n.Message{ID: "OpRecordConfigLogin", Other: "更新登录配置"} - OpRecordConfigOAuth2 = &i18n.Message{ID: "OpRecordConfigOAuth2", Other: "更新OAuth2配置"} - OpRecordConfigLDAP = &i18n.Message{ID: "OpRecordConfigLDAP", Other: "更新LDAP配置"} - OpRecordConfigSMTP = &i18n.Message{ID: "OpRecordConfigSMTP", Other: "更新SMTP配置"} - OpRecordConfigWechat = &i18n.Message{ID: "OpRecordConfigWechat", Other: "更新企业微信配置"} - OpRecordConfigFeishu = &i18n.Message{ID: "OpRecordConfigFeishu", Other: "更新飞书配置"} - OpRecordConfigWebhook = &i18n.Message{ID: "OpRecordConfigWebhook", Other: "更新Webhook配置"} - OpRecordConfigSms = &i18n.Message{ID: "OpRecordConfigSms", Other: "更新短信配置"} - OpRecordConfigSystemVariables = &i18n.Message{ID: "OpRecordConfigSystemVariables", Other: "更新系统变量配置"} - OpRecordConfigCompanyNotice = &i18n.Message{ID: "OpRecordConfigCompanyNotice", Other: "更新系统公告"} - OpRecordDataExportCreate = &i18n.Message{ID: "OpRecordDataExportCreate", Other: "创建数据导出工单"} - OpRecordDataExportCreateWithName = &i18n.Message{ID: "OpRecordDataExportCreateWithName", Other: "创建数据导出工单 %s"} - OpRecordDataExportApproveWithName = &i18n.Message{ID: "OpRecordDataExportApproveWithName", Other: "审批通过数据导出工单 %s"} - OpRecordDataExportRejectWithName = &i18n.Message{ID: "OpRecordDataExportRejectWithName", Other: "驳回数据导出工单 %s"} - OpRecordDataExportExportWithName = &i18n.Message{ID: "OpRecordDataExportExportWithName", Other: "执行数据导出 %s"} - OpRecordDataExportCancel = &i18n.Message{ID: "OpRecordDataExportCancel", Other: "取消数据导出工单"} - OpRecordDataExportCancelWithName = &i18n.Message{ID: "OpRecordDataExportCancelWithName", Other: "取消数据导出工单 %s"} - OpRecordUserUpdateBWP = &i18n.Message{ID: "OpRecordUserUpdateBWP", Other: "修改用户 %s 的业务写权：%s -> %s"} + OpRecordUserCreate = &i18n.Message{ID: "OpRecordUserCreate", Other: "创建用户"} + OpRecordUserCreateWithName = &i18n.Message{ID: "OpRecordUserCreateWithName", Other: "创建用户 %s"} + OpRecordCurrentUserUpdate = &i18n.Message{ID: "OpRecordCurrentUserUpdate", Other: "更新个人中心账号基本信息"} + OpRecordUserUpdate = &i18n.Message{ID: "OpRecordUserUpdate", Other: "更新用户 %s"} + OpRecordUserDelete = &i18n.Message{ID: "OpRecordUserDelete", Other: "删除用户 %s"} + OpRecordMemberCreate = &i18n.Message{ID: "OpRecordMemberCreate", Other: "添加成员"} + OpRecordMemberCreateWithName = &i18n.Message{ID: "OpRecordMemberCreateWithName", Other: "添加成员 %s"} + OpRecordMemberUpdate = &i18n.Message{ID: "OpRecordMemberUpdate", Other: "更新成员 %s"} + OpRecordMemberDelete = &i18n.Message{ID: "OpRecordMemberDelete", Other: "删除成员 %s"} + OpRecordMemberGroupCreate = &i18n.Message{ID: "OpRecordMemberGroupCreate", Other: "添加成员组"} + OpRecordMemberGroupCreateWithName = &i18n.Message{ID: "OpRecordMemberGroupCreateWithName", Other: "添加成员组 %s"} + OpRecordMemberGroupUpdate = &i18n.Message{ID: "OpRecordMemberGroupUpdate", Other: "更新成员组 %s"} + OpRecordMemberGroupDelete = &i18n.Message{ID: "OpRecordMemberGroupDelete", Other: "删除成员组 %s"} + OpRecordRoleCreate = &i18n.Message{ID: "OpRecordRoleCreate", Other: "创建角色"} + OpRecordRoleCreateWithName = &i18n.Message{ID: "OpRecordRoleCreateWithName", Other: "创建角色 %s"} + OpRecordRoleUpdate = &i18n.Message{ID: "OpRecordRoleUpdate", Other: "更新角色 %s"} + OpRecordRoleDelete = &i18n.Message{ID: "OpRecordRoleDelete", Other: "删除角色 %s"} + OpRecordProjectCreate = &i18n.Message{ID: "OpRecordProjectCreate", Other: "创建项目"} + OpRecordProjectCreateWithName = &i18n.Message{ID: "OpRecordProjectCreateWithName", Other: "创建项目 %s"} + OpRecordProjectUpdate = &i18n.Message{ID: "OpRecordProjectUpdate", Other: "更新项目 %s"} + OpRecordProjectDelete = &i18n.Message{ID: "OpRecordProjectDelete", Other: "删除项目 %s"} + OpRecordProjectArchive = &i18n.Message{ID: "OpRecordProjectArchive", Other: "归档项目 %s"} + OpRecordProjectUnarchive = &i18n.Message{ID: "OpRecordProjectUnarchive", Other: "取消归档项目 %s"} + OpRecordDBServiceCreate = &i18n.Message{ID: "OpRecordDBServiceCreate", Other: "创建数据源"} + OpRecordDBServiceCreateWithName = &i18n.Message{ID: "OpRecordDBServiceCreateWithName", Other: "创建数据源 %s"} + OpRecordDBServiceUpdate = &i18n.Message{ID: "OpRecordDBServiceUpdate", Other: "更新数据源 %s"} + OpRecordDBServiceDelete = &i18n.Message{ID: "OpRecordDBServiceDelete", Other: "删除数据源 %s"} + OpRecordDBServiceImport = &i18n.Message{ID: "OpRecordDBServiceImport", Other: "导入数据源"} + OpRecordConfigLogin = &i18n.Message{ID: "OpRecordConfigLogin", Other: "更新登录配置"} + OpRecordConfigOAuth2 = &i18n.Message{ID: "OpRecordConfigOAuth2", Other: "更新OAuth2配置"} + OpRecordConfigLDAP = &i18n.Message{ID: "OpRecordConfigLDAP", Other: "更新LDAP配置"} + OpRecordConfigSMTP = &i18n.Message{ID: "OpRecordConfigSMTP", Other: "更新SMTP配置"} + OpRecordConfigWechat = &i18n.Message{ID: "OpRecordConfigWechat", Other: "更新企业微信配置"} + OpRecordConfigFeishu = &i18n.Message{ID: "OpRecordConfigFeishu", Other: "更新飞书配置"} + OpRecordConfigWebhook = &i18n.Message{ID: "OpRecordConfigWebhook", Other: "更新Webhook配置"} + OpRecordConfigSms = &i18n.Message{ID: "OpRecordConfigSms", Other: "更新短信配置"} + OpRecordConfigSystemVariables = &i18n.Message{ID: "OpRecordConfigSystemVariables", Other: "更新系统变量配置"} + OpRecordConfigCompanyNotice = &i18n.Message{ID: "OpRecordConfigCompanyNotice", Other: "更新系统公告"} + OpRecordDataExportCreate = &i18n.Message{ID: "OpRecordDataExportCreate", Other: "创建数据导出工单"} + OpRecordDataExportCreateWithName = &i18n.Message{ID: "OpRecordDataExportCreateWithName", Other: "创建数据导出工单 %s"} + OpRecordDataExportApproveWithName = &i18n.Message{ID: "OpRecordDataExportApproveWithName", Other: "审批通过数据导出工单 %s"} + OpRecordDataExportRejectWithName = &i18n.Message{ID: "OpRecordDataExportRejectWithName", Other: "驳回数据导出工单 %s"} + OpRecordDataExportExportWithName = &i18n.Message{ID: "OpRecordDataExportExportWithName", Other: "执行数据导出 %s"} + OpRecordDataExportCancel = &i18n.Message{ID: "OpRecordDataExportCancel", Other: "取消数据导出工单"} + OpRecordDataExportCancelWithName = &i18n.Message{ID: "OpRecordDataExportCancelWithName", Other: "取消数据导出工单 %s"} + OpRecordUserUpdateBWP = &i18n.Message{ID: "OpRecordUserUpdateBWP", Other: "修改用户 %s 的业务写权：%s -> %s"} + OpRecordUnmaskingAPICreate = &i18n.Message{ID: "OpRecordUnmaskingAPICreate", Other: "提交查看原文工单申请"} + OpRecordUnmaskingSubmitWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingSubmitWithWorkflowUID", Other: "提交查看原文工单 %s"} + OpRecordUnmaskingApproveWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingApproveWithWorkflowUID", Other: "审批通过查看原文工单 %s"} + OpRecordUnmaskingRejectWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingRejectWithWorkflowUID", Other: "驳回查看原文工单 %s"} + OpRecordUnmaskingViewDetailWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingViewDetailWithWorkflowUID", Other: "查看查看原文工单详情 %s"} + OpRecordUnmaskingViewOriginalWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingViewOriginalWithWorkflowUID", Other: "查看原文（SQL）%s"} + OpRecordUnmaskingDownloadOriginalWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingDownloadOriginalWithWorkflowUID", Other: "下载原文数据 %s"} + OpRecordUnmaskingCancelWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingCancelWithWorkflowUID", Other: "撤回查看原文工单 %s"} + OpRecordUnmaskingUnknownWithWorkflowUID = &i18n.Message{ID: "OpRecordUnmaskingUnknownWithWorkflowUID", Other: "查看原文工单操作 %s (%s)"} +) + +// Unmasking Workflow +var ( + SqlWorkbenchUnmaskingNoPermissionErr = &i18n.Message{ID: "SqlWorkbenchUnmaskingNoPermissionErr", Other: "您没有查看原文的权限，请提交查看原文工单"} + SqlWorkbenchUnmaskingWorkflowNotFoundErr = &i18n.Message{ID: "SqlWorkbenchUnmaskingWorkflowNotFoundErr", Other: "未找到对应的查看原文工单"} ) From ac49b5949e29b22cc121a6308b8a7e424ce62537 Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Wed, 22 Apr 2026 09:15:52 +0000 Subject: [PATCH 11/27] ce feat: add build verification script for multiple edition combinations Introduced a new script to verify that the application can be built successfully under four different GO_BUILD_TAGS combinations: Community, Trial, Enterprise, and DMS Enterprise. This addition enhances the build process by ensuring compatibility across editions before submission. --- Makefile | 4 ++ scripts/verify_build_editions.sh | 102 +++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100755 scripts/verify_build_editions.sh diff --git a/Makefile b/Makefile index 824691b8..ffee44a8 100644 --- a/Makefile +++ b/Makefile @@ -185,6 +185,10 @@ dms_unit_test_clean: dms_test_dms: go test -v -p 1 ./internal/dms/... +# 提交前校验：社区版 / 试用版 / 企业版 / DMS 企业版 四种 GO_BUILD_TAGS 组合下均能 make install +verify_edition_builds: + bash ./scripts/verify_build_editions.sh + ############################### generate ################################## gen_repo_fields: go run ./internal/dms/cmd/gencli/gencli.go -d generate-node-repo-fields ./internal/dms/storage/model/ ./internal/dms/biz/ diff --git a/scripts/verify_build_editions.sh b/scripts/verify_build_editions.sh new file mode 100755 index 00000000..4e238610 --- /dev/null +++ b/scripts/verify_build_editions.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# 提交前校验：社区版 / 试用版 / 企业版 / DMS 企业版 四种构建标签组合均能成功编译（与 Makefile install 一致）。 +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +readonly RED=$'\033[0;31m' +readonly GREEN=$'\033[0;32m' +readonly YELLOW=$'\033[1;33m' +readonly BLUE=$'\033[0;34m' +readonly CYAN=$'\033[0;36m' +readonly BOLD=$'\033[1m' +readonly DIM=$'\033[2m' +readonly NC=$'\033[0m' + +bar() { + printf '%b\n' "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +} + +sub() { + printf '%b %s\n' "${DIM}" "$1${NC}" +} + +err() { + printf '%b %s\n' "${RED}" "$1${NC}" +} + +step_begin() { + local idx="$1" + local total="$2" + local name="$3" + local tags="$4" + printf '\n' + bar + printf '%b Step %s/%s %s\n' "${BOLD}" "$idx" "$total" "$name${NC}" + printf '%b 构建标签 (GO_BUILD_TAGS): %s%s\n' "${CYAN}" "$tags" "${NC}" + bar + printf '%b ⏳ 正在执行 make install …%s\n' "${YELLOW}" "${NC}" +} + +step_ok() { + local secs="$1" + printf '%b ✅ 本步编译成功%s' "${GREEN}" "${NC}" + printf '%b （耗时 %ss）%s\n' "${DIM}" "$secs" "${NC}" +} + +TOTAL_STEPS=4 +FAILED=0 + +run_one() { + local idx="$1" name="$2" tags="$3" + shift 3 + local start=$SECONDS + step_begin "$idx" "$TOTAL_STEPS" "$name" "$tags" + sub "命令: make install $*" + if make install "$@"; then + step_ok "$((SECONDS - start))" + else + err "❌ 本步失败: ${name}" + FAILED=1 + return 1 + fi +} + +printf '\n%b\n' "${BOLD}${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}" +printf '%b\n' "${BOLD}${BLUE}║ 🔍 DMS 多版本构建校验 ║${NC}" +printf '%b\n' "${BOLD}${BLUE}║ 依次验证 Makefile install 在四种标签组合下均可通过 ║${NC}" +printf '%b\n' "${BOLD}${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}" +sub "工作目录: ${ROOT}" + +# 1 社区版: dummyhead（EDITION 默认 ce） +run_one 1 "社区版 (CE)" "dummyhead" || true + +# 2 试用版: dummyhead + trial +if [[ "$FAILED" -eq 0 ]]; then + run_one 2 "试用版 (Trial)" "dummyhead,trial" EDITION=trial || true +fi + +# 3 企业版: dummyhead + enterprise +if [[ "$FAILED" -eq 0 ]]; then + run_one 3 "企业版 (EE)" "dummyhead,enterprise" EDITION=ee || true +fi + +# 4 DMS 企业版: dummyhead + enterprise + dms +if [[ "$FAILED" -eq 0 ]]; then + run_one 4 "DMS 企业版 (EE + DMS)" "dummyhead,enterprise,dms" EDITION=ee PRODUCT_CATEGORY=dms || true +fi + +printf '\n' +bar +if [[ "$FAILED" -eq 0 ]]; then + printf '%b🎉 全部 %s 个版本构建校验通过。%s\n' "${GREEN}${BOLD}" "$TOTAL_STEPS" "${NC}" + bar + printf '\n' + exit 0 +else + printf '%b💥 构建校验未全部通过，请根据上方日志修复后重试。%s\n' "${RED}${BOLD}" "${NC}" + bar + printf '\n' + exit 1 +fi From 3c91773e99a2b6ed77fa9dc25571e037113be4ae Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Sat, 9 May 2026 07:10:59 +0000 Subject: [PATCH 12/27] fix: update ListMaskingRulesReq to clarify projectUid parameter usage Modified the ListMaskingRulesReq struct in masking.go to enhance documentation for the projectUid parameter. The comment now specifies that projectUid is injected via path for specific requests and is optional for global requests, improving clarity for API users. --- api/dms/service/v1/masking.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/dms/service/v1/masking.go b/api/dms/service/v1/masking.go index 94f60bb8..d991780e 100644 --- a/api/dms/service/v1/masking.go +++ b/api/dms/service/v1/masking.go @@ -8,11 +8,10 @@ import ( // swagger:parameters ListMaskingRules type ListMaskingRulesReq struct { - // project uid + // project uid（项目路径下由 path 注入；全局 GET /v1/dms/masking/rules 可不传，仅返回内置规则） // in: path - // Required: true // Example: "project_uid" - ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + ProjectUid string `param:"project_uid" query:"project_uid" json:"project_uid"` // 规则来源筛选: builtin 或 custom，为空时返回全部 // in: query // Example: "custom" From 2fb1655e724b5b48f848bfa1a7d2119a13a50061 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Sat, 9 May 2026 10:00:54 +0000 Subject: [PATCH 13/27] refactor: update ApproveUnmaskingWorkflowReq to make approve_unmasking_workflow optional Modified the ApproveUnmaskingWorkflowReq struct to change the approve_unmasking_workflow field from required to optional. Updated related documentation to reflect this change, clarifying that the field now carries an optional approve_reason. This adjustment improves API flexibility and aligns with user needs. --- api/dms/service/v1/unmasking_workflow.go | 261 ++++++++++++++++++ .../apiserver/service/data_mask_controller.go | 4 +- 2 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 api/dms/service/v1/unmasking_workflow.go diff --git a/api/dms/service/v1/unmasking_workflow.go b/api/dms/service/v1/unmasking_workflow.go new file mode 100644 index 00000000..baff7685 --- /dev/null +++ b/api/dms/service/v1/unmasking_workflow.go @@ -0,0 +1,261 @@ +package v1 + +import ( + "github.com/actiontech/dms/internal/data_masking/biz" + base "github.com/actiontech/dms/pkg/dms-common/api/base/v1" +) + +// swagger:parameters ListUnmaskingWorkflows +type ListUnmaskingWorkflowsReq struct { + // project id + // Required: true + // in: path + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // the maximum count of workflows to be returned + // in: query + // Required: true + PageSize uint32 `query:"page_size" json:"page_size" validate:"required"` + // the offset of workflows to be returned, default is 0 + // in: query + PageIndex uint32 `query:"page_index" json:"page_index"` + // filter the approval status + // in: query + FilterByApprovalStatus biz.UnmaskingWorkflowApprovalStatus `query:"filter_by_approval_status" json:"filter_by_approval_status"` + // filter the usage status + // in: query + FilterByUsageStatus biz.UnmaskingWorkflowUsageStatus `query:"filter_by_usage_status" json:"filter_by_usage_status"` + // filter db_service id + // in: query + FilterByDBServiceUid string `query:"filter_by_db_service_uid" json:"filter_by_db_service_uid"` +} + +// swagger:model ListUnmaskingWorkflowsReply +type ListUnmaskingWorkflowsReply struct { + Data []*UnmaskingWorkflowListItem `json:"data"` + Total int64 `json:"total_nums"` + // Generic reply + base.GenericResp +} + +// swagger:model UnmaskingWorkflowListItem +type UnmaskingWorkflowListItem struct { + // 申请编号 + WorkflowID string `json:"workflow_id"` + // 申请人用户名 + ApplicantName string `json:"applicant_name"` + // 申请时间 (RFC3339) + CreatedAt string `json:"created_at" example:"2024-01-15T10:30:00Z"` + // 数据源实例名称 + DatasourceName string `json:"datasource_name"` + // 数据源实例ID + DatasourceUid string `json:"datasource_uid"` + // 来源类型 + SourceType biz.UnmaskingWorkflowSourceType `json:"source_type" validate:"oneof=data_export sql_workbench"` + // 来源对象UID + SourceUID string `json:"source_uid"` + // 审批状态 + ApprovalStatus biz.UnmaskingWorkflowApprovalStatus `json:"approval_status" validate:"oneof=pending approved rejected cancelled"` + // 使用情况 + UsageStatus biz.UnmaskingWorkflowUsageStatus `json:"usage_status" validate:"oneof=unviewed viewed"` + // 过期时间 (RFC3339) + ExpireTime string `json:"expire_time" example:"2024-01-16T10:30:00Z"` + // 申请理由 + ApplyReason string `json:"apply_reason"` + // 当前待处理人 + CurrentAssignees []*UidWithName `json:"current_assignees"` +} + +// swagger:model CreateUnmaskingWorkflowReq +type CreateUnmaskingWorkflowReq struct { + // swagger:ignore + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // in: body + // Required: true + UnmaskingWorkflow *CreateUnmaskingWorkflow `json:"unmasking_workflow" validate:"required"` +} + +// swagger:model CreateUnmaskingWorkflow +type CreateUnmaskingWorkflow struct { + // 数据源 UID + DatasourceUID string `json:"datasource_uid" validate:"required"` + // SQL 默认 schema + DefaultSchema string `json:"default_schema" validate:"required"` + // 来源类型 + SourceType biz.UnmaskingWorkflowSourceType `json:"source_type" validate:"required,oneof=data_export sql_workbench"` + // 来源对象 UID (如数据导出任务 UID) + SourceUID string `json:"source_uid"` + // 申请理由 + ApplyReason string `json:"apply_reason" validate:"required"` + // 待脱敏 SQL 列表 + UnmaskingSQLs []CreateUnmaskingSQLItem `json:"unmasking_sqls" validate:"required,gt=0"` +} + +// swagger:model CreateUnmaskingSQLItem +type CreateUnmaskingSQLItem struct { + // 来源侧 SQL 索引 id（如数据导出记录中的 SQL 序号）；与 SQL 工作台场景的索引约定可能不同，需结合 source_type、source_uid 解析 + SQLIndexID string `json:"sql_index_id" validate:"required"` + // 原始 SQL 内容 + SQLContent string `json:"sql_content" validate:"required"` +} + +// swagger:model CreateUnmaskingWorkflowReply +type CreateUnmaskingWorkflowReply struct { + Data *CreateUnmaskingWorkflowReplyData `json:"data"` + // Generic reply + base.GenericResp +} + +// swagger:model CreateUnmaskingWorkflowReplyData +type CreateUnmaskingWorkflowReplyData struct { + WorkflowID string `json:"workflow_id"` +} + +// swagger:parameters GetUnmaskingWorkflow +type GetUnmaskingWorkflowReq struct { + // project id + // Required: true + // in: path + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // in: path + // Required: true + WorkflowID string `param:"workflow_id" json:"workflow_id" validate:"required"` +} + +// swagger:model GetUnmaskingWorkflowReply +type GetUnmaskingWorkflowReply struct { + Data *UnmaskingWorkflowDetail `json:"data"` + // Generic reply + base.GenericResp +} + +// swagger:model UnmaskingWorkflowDetail +type UnmaskingWorkflowDetail struct { + UnmaskingWorkflowListItem + // 驳回理由 (整单驳回时) + RejectReason string `json:"reject_reason"` + // 当前待处理人 + CurrentAssignees []*UidWithName `json:"current_assignees"` + // SQL 详情列表 + UnmaskingSQLs []*UnmaskingSQLDetail `json:"unmasking_sqls"` + // 操作日志 + OperationLogs []*UnmaskingOperationLogItem `json:"operation_logs"` +} + +// swagger:model UnmaskingSQLDetail +type UnmaskingSQLDetail struct { + // SQL 详情 UID + UID string `json:"uid"` + // 来源侧 SQL 索引 id + SQLIndexID string `json:"sql_index_id"` + // 原始 SQL 内容 + SQLContent string `json:"sql_content"` + // 脱敏配置快照 + MaskingConfigSnapshot []*biz.ColumnMaskingConfig `json:"masking_config_snapshot,omitempty"` + // 血缘分析快照 + LineageAnalysisSnapshot *biz.AnalyzeResult `json:"lineage_analysis_snapshot,omitempty"` + + UnmaskingSQLPreviewData +} + +// swagger:model UnmaskingSQLPreviewData +type UnmaskingSQLPreviewData struct { + // 脱敏后的预览数据 (普通用户仅能看到此数据) + MaskedData *SQLQueryResult `json:"masked_data"` + // 原始采样数据 (仅有权限的审核人能看到) + OriginalData *SQLQueryResult `json:"original_data"` +} + +// swagger:model SQLQueryResultRow +// SQLQueryResultRow 一行数据；每个元素为单元格的字符串形式（与 columns 顺序一致）。 +type SQLQueryResultRow []string + +// swagger:model SQLQueryResult +type SQLQueryResult struct { + // 列名列表 + Columns []string `json:"columns"` + // 数据行列表 (每一行的数据顺序与 Columns 一致) + Rows []SQLQueryResultRow `json:"rows"` + // 总行数 + RowCount int `json:"row_count"` +} + +// swagger:model UnmaskingOperationLogItem +type UnmaskingOperationLogItem struct { + // 操作人 UID + OperatorUID string `json:"operator_uid"` + // 操作人姓名 + OperatorName string `json:"operator_name"` + // 操作动作 + Action biz.UnmaskingAction `json:"action"` + // 操作时间 (RFC3339) + ActionTime string `json:"action_time"` + // 额外信息 (如拦截原因) + ExtraMessage string `json:"extra_message"` +} + +// swagger:model ApproveUnmaskingWorkflowReq +type ApproveUnmaskingWorkflowReq struct { + // swagger:ignore + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // in: path + // Required: true + // swagger:ignore + WorkflowID string `param:"workflow_id" json:"workflow_id" validate:"required"` + // in: body + ApproveUnmaskingWorkflow *ApproveUnmaskingWorkflow `json:"approve_unmasking_workflow,omitempty"` +} + +// swagger:model ApproveUnmaskingWorkflow +type ApproveUnmaskingWorkflow struct { + // 审批理由 非必须 + ApproveReason string `json:"approve_reason"` +} + +// swagger:model ApproveUnmaskingWorkflowReply +type ApproveUnmaskingWorkflowReply struct { + // Generic reply + base.GenericResp +} + +// swagger:model RejectUnmaskingWorkflowReq +type RejectUnmaskingWorkflowReq struct { + // swagger:ignore + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // in: path + // Required: true + // swagger:ignore + WorkflowID string `param:"workflow_id" json:"workflow_id" validate:"required"` + // in: body + // Required: true + RejectUnmaskingWorkflow *RejectUnmaskingWorkflow `json:"reject_unmasking_workflow" validate:"required"` +} + +// swagger:model RejectUnmaskingWorkflow +type RejectUnmaskingWorkflow struct { + // 驳回理由 + // Required: true + RejectReason string `json:"reject_reason" validate:"required"` +} + +// swagger:model RejectUnmaskingWorkflowReply +type RejectUnmaskingWorkflowReply struct { + // Generic reply + base.GenericResp +} + +// swagger:parameters CancelUnmaskingWorkflow +type CancelUnmaskingWorkflowReq struct { + // project id + // Required: true + // in: path + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // in: path + // Required: true + WorkflowID string `param:"workflow_id" json:"workflow_id" validate:"required"` +} + +// swagger:model CancelUnmaskingWorkflowReply +type CancelUnmaskingWorkflowReply struct { + // Generic reply + base.GenericResp +} diff --git a/internal/apiserver/service/data_mask_controller.go b/internal/apiserver/service/data_mask_controller.go index 65b0d167..45054c5e 100644 --- a/internal/apiserver/service/data_mask_controller.go +++ b/internal/apiserver/service/data_mask_controller.go @@ -680,9 +680,9 @@ func (ctl *DMSController) GetUnmaskingWorkflow(c echo.Context) error { // required: true // type: string // - name: approve_unmasking_workflow -// description: approve unmasking workflow info +// description: approve unmasking workflow info (optional, only carries approve_reason) // in: body -// required: true +// required: false // schema: // "$ref": "#/definitions/ApproveUnmaskingWorkflow" // From cc4752e533a91f24bd2a48b50a429679ed194b4c Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Sat, 9 May 2026 10:19:59 +0000 Subject: [PATCH 14/27] refactor: improve SQL audit middleware error handling and request parsing Updated the AuditMiddleware function to enhance error handling by logging warnings instead of errors when parsing SQL requests fails or when SQL and datasource ID are empty. This change ensures that audit processing does not block user SQL execution. Additionally, modified the base64 decoding to use URL-safe encoding, improving compatibility with session IDs generated by the ODC server. --- .../service/sql_workbench_service.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/sql_workbench/service/sql_workbench_service.go b/internal/sql_workbench/service/sql_workbench_service.go index 9e3d4913..f237ced1 100644 --- a/internal/sql_workbench/service/sql_workbench_service.go +++ b/internal/sql_workbench/service/sql_workbench_service.go @@ -1056,15 +1056,18 @@ func (sqlWorkbenchService *SqlWorkbenchService) AuditMiddleware() echo.Middlewar c.Request().Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) // 解析请求体获取 SQL 和 datasource ID + // 注意：解析仅服务于审核辅助路径，解析失败不应直接阻塞用户的 SQL 执行； + // 否则一旦中间件辅助能力出错（如 sid 解码失败），用户连查询都跑不了。 + // 真正的「未启用审核 / 审核失败」等强策略仍由后续分支按既有 fail-closed 处理。 sql, datasourceID, err := sqlWorkbenchService.parseStreamExecuteRequest(bodyBytes) if err != nil { - sqlWorkbenchService.log.Errorf("failed to parse streamExecute request, skipping audit: %v", err) - return errors.New(locale.Bundle.LocalizeMsgByCtx(c.Request().Context(), locale.SqlWorkbenchAuditParseReqErr)) + sqlWorkbenchService.log.Warnf("failed to parse streamExecute request, skipping audit: %v", err) + return next(c) } if sql == "" || datasourceID == "" { - sqlWorkbenchService.log.Debugf("SQL or datasource ID is empty, skipping audit") - return errors.New(locale.Bundle.LocalizeMsgByCtx(c.Request().Context(), locale.SqlWorkbenchAuditMissingSQLOrDatasourceErr)) + sqlWorkbenchService.log.Warnf("SQL or datasource ID is empty, skipping audit") + return next(c) } // 获取当前用户 ID @@ -1159,8 +1162,9 @@ func (sqlWorkbenchService *SqlWorkbenchService) parseSidToDatasourceID(sid strin sid = sid[:idx] } - // 解码 base64 - decodedBytes, err := base64.StdEncoding.DecodeString(sid) + // ODC 服务端使用 Base64.getUrlEncoder() 生成 sessionId（URL-safe，包含 '-'/'_'）， + // 这里必须用 URLEncoding 解码，否则遇到 '-'/'_' 会报 illegal base64 data。 + decodedBytes, err := base64.URLEncoding.DecodeString(sid) if err != nil { return "", fmt.Errorf("failed to decode base64 sid: %v", err) } From 2a7fb1b5b614268a3cb35e1a03f34b0b21d893bc Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Mon, 11 May 2026 11:27:37 +0000 Subject: [PATCH 15/27] refactor: streamline Cloudbeaver and DataExportWorkflow usecases by removing maskingTaskRepo Removed the maskingTaskRepo from the CloudbeaverUsecase and DataExportWorkflowUsecase, replacing its functionality with the new HasSensitiveDataMaskingTask method in the DBServiceUsecase. This change simplifies the usecases and enhances the handling of sensitive data masking tasks, improving overall code clarity and maintainability. --- internal/dms/biz/cloudbeaver.go | 12 +++++------ internal/dms/biz/data_export_workflow.go | 6 +++--- internal/dms/biz/db_service.go | 20 +++++++++++++++++++ internal/dms/service/cloudbeaver.go | 2 +- internal/dms/service/service.go | 2 +- .../service/data_masking_middleware.go | 2 +- 6 files changed, 31 insertions(+), 13 deletions(-) diff --git a/internal/dms/biz/cloudbeaver.go b/internal/dms/biz/cloudbeaver.go index 25eb5e11..4488fb83 100644 --- a/internal/dms/biz/cloudbeaver.go +++ b/internal/dms/biz/cloudbeaver.go @@ -91,13 +91,12 @@ type CloudbeaverUsecase struct { sqlResultMasker SQLResultMasker cbOperationLogUsecase *CbOperationLogUsecase projectUsecase *ProjectUsecase - maskingTaskRepo MaskingTaskRepo repo CloudbeaverRepo proxyTargetRepo ProxyTargetRepo maintenanceTimeUsecase *MaintenanceTimeUsecase } -func NewCloudbeaverUsecase(log utilLog.Logger, cfg *CloudbeaverCfg, userUsecase *UserUsecase, dbServiceUsecase *DBServiceUsecase, opPermissionVerifyUsecase *OpPermissionVerifyUsecase, dmsConfigUseCase *DMSConfigUseCase, sqlResultMasker SQLResultMasker, cloudbeaverRepo CloudbeaverRepo, proxyTargetRepo ProxyTargetRepo, cbOperationUseCase *CbOperationLogUsecase, projectUsecase *ProjectUsecase, maskingTaskRepo MaskingTaskRepo, maintenanceTimeUsecase *MaintenanceTimeUsecase) (cu *CloudbeaverUsecase) { +func NewCloudbeaverUsecase(log utilLog.Logger, cfg *CloudbeaverCfg, userUsecase *UserUsecase, dbServiceUsecase *DBServiceUsecase, opPermissionVerifyUsecase *OpPermissionVerifyUsecase, dmsConfigUseCase *DMSConfigUseCase, sqlResultMasker SQLResultMasker, cloudbeaverRepo CloudbeaverRepo, proxyTargetRepo ProxyTargetRepo, cbOperationUseCase *CbOperationLogUsecase, projectUsecase *ProjectUsecase, maintenanceTimeUsecase *MaintenanceTimeUsecase) (cu *CloudbeaverUsecase) { cu = &CloudbeaverUsecase{ repo: cloudbeaverRepo, proxyTargetRepo: proxyTargetRepo, @@ -108,7 +107,6 @@ func NewCloudbeaverUsecase(log utilLog.Logger, cfg *CloudbeaverCfg, userUsecase sqlResultMasker: sqlResultMasker, cbOperationLogUsecase: cbOperationUseCase, projectUsecase: projectUsecase, - maskingTaskRepo: maskingTaskRepo, cloudbeaverCfg: cfg, log: utilLog.NewHelper(log, utilLog.WithMessageKey("biz.cloudbeaver")), maintenanceTimeUsecase: maintenanceTimeUsecase, @@ -467,7 +465,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { return err } - isMaskingEnabled, _ := cu.maskingTaskRepo.CheckMaskingTaskExist(c.Request().Context(), dbService.UID) + isMaskingEnabled := cu.dbServiceUsecase != nil && cu.dbServiceUsecase.HasSensitiveDataMaskingTask(c.Request().Context(), dbService.UID) // 构建任务ID与数据脱敏的关联 return cu.buildTaskIdAssocDataMasking(cloudbeaverResBuf.Bytes(), taskMaskingContext{ Enabled: isMaskingEnabled, @@ -497,7 +495,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { cu.log.Error(err) } - isMaskingEnabled, _ := cu.maskingTaskRepo.CheckMaskingTaskExist(c.Request().Context(), dbService.UID) + isMaskingEnabled := cu.dbServiceUsecase != nil && cu.dbServiceUsecase.HasSensitiveDataMaskingTask(c.Request().Context(), dbService.UID) maskCtx := taskMaskingContext{ Enabled: isMaskingEnabled, DBServiceUID: dbService.UID, @@ -567,7 +565,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { return err } - isMaskingEnabled, _ := cu.maskingTaskRepo.CheckMaskingTaskExist(c.Request().Context(), dbService.UID) + isMaskingEnabled := cu.dbServiceUsecase != nil && cu.dbServiceUsecase.HasSensitiveDataMaskingTask(c.Request().Context(), dbService.UID) // 构建任务ID与数据脱敏的关联 return cu.buildTaskIdAssocDataMasking(cloudbeaverResBuf.Bytes(), taskMaskingContext{ Enabled: isMaskingEnabled, @@ -702,7 +700,7 @@ func (cu *CloudbeaverUsecase) GraphQLDistributor() echo.MiddlewareFunc { } if params.OperationName == "asyncSqlExecuteQuery" { - isMaskingEnabled, _ := cu.maskingTaskRepo.CheckMaskingTaskExist(c.Request().Context(), dbService.UID) + isMaskingEnabled := cu.dbServiceUsecase != nil && cu.dbServiceUsecase.HasSensitiveDataMaskingTask(c.Request().Context(), dbService.UID) if err := cu.buildTaskIdAssocDataMasking(cloudbeaverResBuf.Bytes(), taskMaskingContext{ Enabled: isMaskingEnabled, DBServiceUID: dbService.UID, diff --git a/internal/dms/biz/data_export_workflow.go b/internal/dms/biz/data_export_workflow.go index a574bedb..be58e706 100644 --- a/internal/dms/biz/data_export_workflow.go +++ b/internal/dms/biz/data_export_workflow.go @@ -136,13 +136,13 @@ type DataExportWorkflowUsecase struct { webhookUsecase *WebHookConfigurationUsecase userUsecase *UserUsecase systemVariableUsecase *SystemVariableUsecase - maskingTaskRepo MaskingTaskRepo + dbServiceUsecase *DBServiceUsecase unmaskingWorkflowUsecase *dataMaskingBiz.UnmaskingWorkflowUsecase log *utilLog.Helper reportHost string } -func NewDataExportWorkflowUsecase(logger utilLog.Logger, tx TransactionGenerator, repo WorkflowRepo, dataExportTaskRepo DataExportTaskRepo, dbServiceRepo DBServiceRepo, maskingConfigRepo DataExportMaskingConfigRepo, maskingRuleRepo DataExportMaskingRuleRepo, opPermissionVerifyUsecase *OpPermissionVerifyUsecase, projectUsecase *ProjectUsecase, proxyTargetRepo ProxyTargetRepo, clusterUseCase *ClusterUsecase, webhookUsecase *WebHookConfigurationUsecase, userUsecase *UserUsecase, systemVariableUsecase *SystemVariableUsecase, maskingTaskRepo MaskingTaskRepo, unmaskingWorkflowUsecase *dataMaskingBiz.UnmaskingWorkflowUsecase, reportHost string) *DataExportWorkflowUsecase { +func NewDataExportWorkflowUsecase(logger utilLog.Logger, tx TransactionGenerator, repo WorkflowRepo, dataExportTaskRepo DataExportTaskRepo, dbServiceRepo DBServiceRepo, maskingConfigRepo DataExportMaskingConfigRepo, maskingRuleRepo DataExportMaskingRuleRepo, opPermissionVerifyUsecase *OpPermissionVerifyUsecase, projectUsecase *ProjectUsecase, proxyTargetRepo ProxyTargetRepo, clusterUseCase *ClusterUsecase, webhookUsecase *WebHookConfigurationUsecase, userUsecase *UserUsecase, systemVariableUsecase *SystemVariableUsecase, dbServiceUsecase *DBServiceUsecase, unmaskingWorkflowUsecase *dataMaskingBiz.UnmaskingWorkflowUsecase, reportHost string) *DataExportWorkflowUsecase { return &DataExportWorkflowUsecase{ tx: tx, repo: repo, @@ -157,7 +157,7 @@ func NewDataExportWorkflowUsecase(logger utilLog.Logger, tx TransactionGenerator webhookUsecase: webhookUsecase, userUsecase: userUsecase, systemVariableUsecase: systemVariableUsecase, - maskingTaskRepo: maskingTaskRepo, + dbServiceUsecase: dbServiceUsecase, unmaskingWorkflowUsecase: unmaskingWorkflowUsecase, log: utilLog.NewHelper(logger, utilLog.WithMessageKey("biz.dataExportWorkflow")), reportHost: reportHost, diff --git a/internal/dms/biz/db_service.go b/internal/dms/biz/db_service.go index 782a56d3..bdc4d5d8 100644 --- a/internal/dms/biz/db_service.go +++ b/internal/dms/biz/db_service.go @@ -201,6 +201,26 @@ func NewDBServiceUsecase(log utilLog.Logger, repo DBServiceRepo, maskingTaskRepo } } +// CheckSensitiveDataMaskingTask 查询 data_masking_discovery_tasks 是否存在该数据源的任务行；错误原样返回（用于导出等需失败路径）。 +func (d *DBServiceUsecase) CheckSensitiveDataMaskingTask(ctx context.Context, dbServiceUID string) (bool, error) { + if d == nil || d.maskingTaskRepo == nil || dbServiceUID == "" { + return false, nil + } + return d.maskingTaskRepo.CheckMaskingTaskExist(ctx, dbServiceUID) +} + +// HasSensitiveDataMaskingTask 该数据源是否已存在敏感数据发现任务；查询语义与 CheckSensitiveDataMaskingTask 一致，存储错误时视为未开启并打 debug 日志。 +func (d *DBServiceUsecase) HasSensitiveDataMaskingTask(ctx context.Context, dbServiceUID string) bool { + ok, err := d.CheckSensitiveDataMaskingTask(ctx, dbServiceUID) + if err != nil { + if d != nil && d.log != nil { + d.log.Debugf("CheckSensitiveDataMaskingTask db_service_uid=%s: %v", dbServiceUID, err) + } + return false + } + return ok +} + type BizDBServiceArgs struct { Name string Desc *string diff --git a/internal/dms/service/cloudbeaver.go b/internal/dms/service/cloudbeaver.go index 15f81d4e..3978fb40 100644 --- a/internal/dms/service/cloudbeaver.go +++ b/internal/dms/service/cloudbeaver.go @@ -82,7 +82,7 @@ func NewAndInitCloudbeaverService(logger utilLog.Logger, opts *conf.DMSOptions) } } - cloudbeaverUsecase := biz.NewCloudbeaverUsecase(logger, cfg, userUsecase, dbServiceUseCase, opPermissionVerifyUsecase, dmsConfigUseCase, sqlResultMasker, cloudbeaverRepo, dmsProxyTargetRepo, cbOperationLogUsecase, projectUsecase, discoveryTaskRepo, maintenanceTimeUsecase) + cloudbeaverUsecase := biz.NewCloudbeaverUsecase(logger, cfg, userUsecase, dbServiceUseCase, opPermissionVerifyUsecase, dmsConfigUseCase, sqlResultMasker, cloudbeaverRepo, dmsProxyTargetRepo, cbOperationLogUsecase, projectUsecase, maintenanceTimeUsecase) proxyUsecase := biz.NewCloudbeaverProxyUsecase(logger, cloudbeaverUsecase) return &CloudbeaverService{ diff --git a/internal/dms/service/service.go b/internal/dms/service/service.go index d69cb01c..f88b4ac4 100644 --- a/internal/dms/service/service.go +++ b/internal/dms/service/service.go @@ -158,7 +158,7 @@ func NewAndInitDMSService(logger utilLog.Logger, opts *conf.DMSOptions) (*DMSSer if err != nil { return nil, fmt.Errorf("failed to initialize unmasking workflow usecase: %v", err) } - DataExportWorkflowUsecase := biz.NewDataExportWorkflowUsecase(logger, tx, workflowRepo, dataExportTaskRepo, dbServiceRepo, dataExportMaskingConfigRepo, dataExportMaskingRuleRepo, opPermissionVerifyUsecase, projectUsecase, dmsProxyTargetRepo, clusterUsecase, webhookConfigurationUsecase, userUsecase, systemVariableUsecase, discoveryTaskRepo, unmaskingWorkflowUsecase, fmt.Sprintf("%s:%d", opts.ReportHost, opts.APIServiceOpts.Port)) + DataExportWorkflowUsecase := biz.NewDataExportWorkflowUsecase(logger, tx, workflowRepo, dataExportTaskRepo, dbServiceRepo, dataExportMaskingConfigRepo, dataExportMaskingRuleRepo, opPermissionVerifyUsecase, projectUsecase, dmsProxyTargetRepo, clusterUsecase, webhookConfigurationUsecase, userUsecase, systemVariableUsecase, dbServiceUseCase, unmaskingWorkflowUsecase, fmt.Sprintf("%s:%d", opts.ReportHost, opts.APIServiceOpts.Port)) dataMaskingUsecase, stopDataMaskingScheduler, err := initDataMaskingUsecase(logger, st, dbServiceUseCase, clusterUsecase, dmsProxyTargetRepo) if err != nil { return nil, fmt.Errorf("failed to initialize data masking usecase: %v", err) diff --git a/internal/sql_workbench/service/data_masking_middleware.go b/internal/sql_workbench/service/data_masking_middleware.go index 46e0dc76..621dca64 100644 --- a/internal/sql_workbench/service/data_masking_middleware.go +++ b/internal/sql_workbench/service/data_masking_middleware.go @@ -1,8 +1,8 @@ package sql_workbench import ( - "github.com/actiontech/dms/internal/dms/biz" dataMaskingBiz "github.com/actiontech/dms/internal/data_masking/biz" + "github.com/actiontech/dms/internal/dms/biz" "github.com/actiontech/dms/internal/sql_workbench/sqlresultmasker" ) From 75b31cb78eb920f1c8fac43a3c11d8ed623de3c2 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Tue, 12 May 2026 07:38:48 +0000 Subject: [PATCH 16/27] fix(api): align data export and db structure column contracts with EE --- api/dms/service/v1/data_export_task.go | 5 +++ api/dms/service/v1/data_export_workflow.go | 15 +++++++++ api/dms/service/v1/db_structure_columns.go | 36 ++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 api/dms/service/v1/db_structure_columns.go diff --git a/api/dms/service/v1/data_export_task.go b/api/dms/service/v1/data_export_task.go index 37b8eb5a..498616d8 100644 --- a/api/dms/service/v1/data_export_task.go +++ b/api/dms/service/v1/data_export_task.go @@ -3,6 +3,7 @@ package v1 import ( "time" + maskingBiz "github.com/actiontech/dms/internal/data_masking/biz" base "github.com/actiontech/dms/pkg/dms-common/api/base/v1" ) @@ -127,6 +128,10 @@ type ListDataExportTaskSQL struct { ExportSQLType string `json:"export_sql_type"` AuditLevel string `json:"audit_level"` AuditSQLResult []AuditSQLResult `json:"audit_sql_result"` + // 血缘分析快照（与查看原文工单 SQL 详情字段语义一致） + LineageAnalysisSnapshot *maskingBiz.AnalyzeResult `json:"lineage_analysis_snapshot,omitempty"` + // 脱敏配置快照 + MaskingConfigSnapshot []*maskingBiz.ColumnMaskingConfig `json:"masking_config_snapshot,omitempty"` } type AuditSQLResult struct { Level string `json:"level" example:"warn"` diff --git a/api/dms/service/v1/data_export_workflow.go b/api/dms/service/v1/data_export_workflow.go index 3e3c8151..c2721ab5 100644 --- a/api/dms/service/v1/data_export_workflow.go +++ b/api/dms/service/v1/data_export_workflow.go @@ -267,6 +267,21 @@ type ExportDataExportWorkflowReq struct { DataExportWorkflowUid string `param:"data_export_workflow_uid" json:"data_export_workflow_uid" validate:"required"` } +// swagger:parameters DownloadOriginalDataExportWorkflow +type DownloadOriginalDataExportWorkflowReq struct { + // project id + // Required: true + // in:path + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // Required: true + // in:path + DataExportWorkflowUid string `param:"data_export_workflow_uid" json:"data_export_workflow_uid" validate:"required"` + // 已批准的查看原文工单 UID + // Required: true + // in:query + UnmaskingWorkflowUid string `query:"unmasking_workflow_uid" json:"unmasking_workflow_uid" validate:"required"` +} + type RejectDataExportWorkflowPayload struct { // Required: true Reason string `json:"reason" validate:"required"` diff --git a/api/dms/service/v1/db_structure_columns.go b/api/dms/service/v1/db_structure_columns.go new file mode 100644 index 00000000..632e694d --- /dev/null +++ b/api/dms/service/v1/db_structure_columns.go @@ -0,0 +1,36 @@ +package v1 + +import ( + base "github.com/actiontech/dms/pkg/dms-common/api/base/v1" +) + +// swagger:parameters ListTableColumns +type ListTableColumnsReq struct { + // Required: true + // in:path + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // Required: true + // in:path + DBServiceUid string `param:"db_service_uid" json:"db_service_uid" validate:"required"` + // Required: true + // in:path + SchemaName string `param:"schema_name" json:"schema_name" validate:"required"` + // Required: true + // in:path + TableName string `param:"table_name" json:"table_name" validate:"required"` +} + +// swagger:model ListTableColumnsReply +type ListTableColumnsReply struct { + Data []*TableColumn `json:"data"` + // Generic reply + base.GenericResp +} + +// swagger:model TableColumn +type TableColumn struct { + Name string `json:"name"` + Type string `json:"type"` + Comment string `json:"comment"` + Nullable bool `json:"nullable"` +} From b18569e6bb0928ec567eed4490e9554bc43d3cbb Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Wed, 13 May 2026 08:13:44 +0000 Subject: [PATCH 17/27] feat(api): integrate SQL Workbench data masking middleware and storage initialization Added data masking middleware for SQL Workbench, including components for data masking and unmasking workflows. Initialized storage for the SQL result masker, ensuring it shares configuration with DMS. This enhances data protection capabilities within the API server. --- internal/apiserver/service/router.go | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/internal/apiserver/service/router.go b/internal/apiserver/service/router.go index 825f1d88..6acdf54d 100644 --- a/internal/apiserver/service/router.go +++ b/internal/apiserver/service/router.go @@ -7,6 +7,8 @@ import ( dmsMiddleware "github.com/actiontech/dms/internal/apiserver/middleware" "github.com/actiontech/dms/internal/dms/biz" + dmsService "github.com/actiontech/dms/internal/dms/service" + "github.com/actiontech/dms/internal/dms/storage" "github.com/actiontech/dms/internal/pkg/locale" "github.com/actiontech/dms/internal/pkg/utils" sqlWorkbenchService "github.com/actiontech/dms/internal/sql_workbench/service" @@ -283,6 +285,21 @@ func (s *APIServer) initRouter() error { sqlWorkbenchV1.Use(s.SqlWorkbenchController.SqlWorkbenchService.Login()) + // 添加数据脱敏中间件 + sqlWorkbenchV1.Use(sqlWorkbenchService.GetDataMaskingMiddleware(sqlWorkbenchService.DataMaskingMiddlewareConfig{ + SqlResultMasker: s.SqlWorkbenchController.SqlWorkbenchService.GetSqlResultMasker(), + DBServiceUsecase: s.DMSController.DMS.DBServiceUsecase, + SqlWorkbenchService: s.SqlWorkbenchController.SqlWorkbenchService, + UnmaskingWorkflowUsecase: s.DMSController.DMS.UnmaskingWorkflowUsecase, + })) + + // 添加查看原文 SQL 替换中间件 + sqlWorkbenchV1.Use(sqlWorkbenchService.GetUnmaskingWorkflowMiddleware(sqlWorkbenchService.DataMaskingMiddlewareConfig{ + DBServiceUsecase: s.DMSController.DMS.DBServiceUsecase, + SqlWorkbenchService: s.SqlWorkbenchController.SqlWorkbenchService, + UnmaskingWorkflowUsecase: s.DMSController.DMS.UnmaskingWorkflowUsecase, + })) + // 添加操作日志记录中间件 sqlWorkbenchV1.Use(sqlWorkbenchService.GetOperationLogBodyDumpMiddleware(sqlWorkbenchService.OperationLogMiddlewareConfig{ CbOperationLogUsecase: s.DMSController.DMS.CbOperationLogUsecase, @@ -485,6 +502,25 @@ func (s *APIServer) installController() error { s.DMSController = DMSController s.SqlWorkbenchController = sqlWorkbenchController + // 初始化 SQL Workbench 脱敏组件，与 DMS 共用同一套存储配置 + st, err := storage.NewStorage(s.logger, &storage.StorageConfig{ + User: s.opts.ServiceOpts.Database.UserName, + Password: s.opts.ServiceOpts.Database.Password, + Host: s.opts.ServiceOpts.Database.Host, + Port: s.opts.ServiceOpts.Database.Port, + Schema: s.opts.ServiceOpts.Database.Database, + Debug: s.opts.ServiceOpts.Database.Debug, + AutoMigrate: s.opts.ServiceOpts.Database.AutoMigrate, + }) + if err != nil { + return fmt.Errorf("failed to initialize storage for masker: %v", err) + } + masker, err := dmsService.NewSQLWorkbenchSQLResultMasker(s.logger, st) + if err != nil { + return fmt.Errorf("failed to create sql result masker: %v", err) + } + s.SqlWorkbenchController.SqlWorkbenchService.SetSqlResultMasker(masker) + // s.AuthController.RegisterPlugin(s.DMSController.GetRegisterPluginFn()) return nil } From 62602a91396c8bdc0305f7b20ff4b66c9e3046d1 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Wed, 13 May 2026 09:54:01 +0000 Subject: [PATCH 18/27] refactor(swagger): update parameter definitions and response schemas for data export and unmasking workflows Modified swagger comments to improve clarity and consistency in API documentation. Adjusted parameter references and response schemas across multiple functions in the DMSController and related files, ensuring accurate representation of request and response structures. --- api/dms/service/v1/data_export_workflow.go | 2 +- .../apiserver/service/data_mask_controller.go | 102 +++++++++--------- internal/apiserver/service/dms_controller.go | 68 +++++++++--- 3 files changed, 106 insertions(+), 66 deletions(-) diff --git a/api/dms/service/v1/data_export_workflow.go b/api/dms/service/v1/data_export_workflow.go index c2721ab5..0273ec6f 100644 --- a/api/dms/service/v1/data_export_workflow.go +++ b/api/dms/service/v1/data_export_workflow.go @@ -41,7 +41,7 @@ type AddDataExportWorkflowReply struct { base.GenericResp } -// swagger:parameters ListDataExportWorkflows ListAllDataExportWorkflows +// swagger:parameters ListDataExportWorkflows type ListDataExportWorkflowsReq struct { // project id // Required: false diff --git a/internal/apiserver/service/data_mask_controller.go b/internal/apiserver/service/data_mask_controller.go index 45054c5e..0c5fdac2 100644 --- a/internal/apiserver/service/data_mask_controller.go +++ b/internal/apiserver/service/data_mask_controller.go @@ -516,18 +516,18 @@ func (ctl *DMSController) GetTableColumnMaskingDetails(c echo.Context) error { // in: body // required: true // schema: -// "$ref": "#/definitions/CreateUnmaskingWorkflowReq" +// "$ref": "#/definitions/CreateUnmaskingWorkflowReq" // // responses: // -// '200': -// description: Create unmasking workflow successfully -// schema: -// "$ref": "#/definitions/CreateUnmaskingWorkflowReply" -// default: -// description: Generic error response -// schema: -// "$ref": "#/definitions/GenericResp" +// '200': +// description: Create unmasking workflow successfully +// schema: +// "$ref": "#/definitions/CreateUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" func (ctl *DMSController) CreateUnmaskingWorkflow(c echo.Context) error { req := &aV1.CreateUnmaskingWorkflowReq{} if err := bindAndValidateReq(c, req); err != nil { @@ -590,14 +590,14 @@ func (ctl *DMSController) CreateUnmaskingWorkflow(c echo.Context) error { // // responses: // -// '200': -// description: List unmasking workflows successfully -// schema: -// "$ref": "#/definitions/ListUnmaskingWorkflowsReply" -// default: -// description: Generic error response -// schema: -// "$ref": "#/definitions/GenericResp" +// '200': +// description: List unmasking workflows successfully +// schema: +// "$ref": "#/definitions/ListUnmaskingWorkflowsReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" func (ctl *DMSController) ListUnmaskingWorkflows(c echo.Context) error { req := &aV1.ListUnmaskingWorkflowsReq{} if err := bindAndValidateReq(c, req); err != nil { @@ -636,14 +636,14 @@ func (ctl *DMSController) ListUnmaskingWorkflows(c echo.Context) error { // // responses: // -// '200': -// description: Get unmasking workflow detail successfully -// schema: -// "$ref": "#/definitions/GetUnmaskingWorkflowReply" -// default: -// description: Generic error response -// schema: -// "$ref": "#/definitions/GenericResp" +// '200': +// description: Get unmasking workflow detail successfully +// schema: +// "$ref": "#/definitions/GetUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" func (ctl *DMSController) GetUnmaskingWorkflow(c echo.Context) error { req := &aV1.GetUnmaskingWorkflowReq{} if err := bindAndValidateReq(c, req); err != nil { @@ -684,18 +684,18 @@ func (ctl *DMSController) GetUnmaskingWorkflow(c echo.Context) error { // in: body // required: false // schema: -// "$ref": "#/definitions/ApproveUnmaskingWorkflow" +// "$ref": "#/definitions/ApproveUnmaskingWorkflow" // // responses: // -// '200': -// description: Approve unmasking workflow successfully -// schema: -// "$ref": "#/definitions/ApproveUnmaskingWorkflowReply" -// default: -// description: Generic error response -// schema: -// "$ref": "#/definitions/GenericResp" +// '200': +// description: Approve unmasking workflow successfully +// schema: +// "$ref": "#/definitions/ApproveUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" func (ctl *DMSController) ApproveUnmaskingWorkflow(c echo.Context) error { req := &aV1.ApproveUnmaskingWorkflowReq{} if err := bindAndValidateReq(c, req); err != nil { @@ -736,18 +736,18 @@ func (ctl *DMSController) ApproveUnmaskingWorkflow(c echo.Context) error { // in: body // required: true // schema: -// "$ref": "#/definitions/RejectUnmaskingWorkflow" +// "$ref": "#/definitions/RejectUnmaskingWorkflow" // // responses: // -// '200': -// description: Reject unmasking workflow successfully -// schema: -// "$ref": "#/definitions/RejectUnmaskingWorkflowReply" -// default: -// description: Generic error response -// schema: -// "$ref": "#/definitions/GenericResp" +// '200': +// description: Reject unmasking workflow successfully +// schema: +// "$ref": "#/definitions/RejectUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" func (ctl *DMSController) RejectUnmaskingWorkflow(c echo.Context) error { req := &aV1.RejectUnmaskingWorkflowReq{} if err := bindAndValidateReq(c, req); err != nil { @@ -786,14 +786,14 @@ func (ctl *DMSController) RejectUnmaskingWorkflow(c echo.Context) error { // // responses: // -// '200': -// description: Cancel unmasking workflow successfully -// schema: -// "$ref": "#/definitions/CancelUnmaskingWorkflowReply" -// default: -// description: Generic error response -// schema: -// "$ref": "#/definitions/GenericResp" +// '200': +// description: Cancel unmasking workflow successfully +// schema: +// "$ref": "#/definitions/CancelUnmaskingWorkflowReply" +// default: +// description: Generic error response +// schema: +// "$ref": "#/definitions/GenericResp" func (ctl *DMSController) CancelUnmaskingWorkflow(c echo.Context) error { req := &aV1.CancelUnmaskingWorkflowReq{} if err := bindAndValidateReq(c, req); err != nil { diff --git a/internal/apiserver/service/dms_controller.go b/internal/apiserver/service/dms_controller.go index 5582b58e..17b9108d 100644 --- a/internal/apiserver/service/dms_controller.go +++ b/internal/apiserver/service/dms_controller.go @@ -86,7 +86,8 @@ func (ctl *DMSController) Shutdown() error { // description: the name of environment tag to be created // in: body // required: true -// type: string +// schema: +// "$ref": "#/definitions/CreateEnvironmentTagReq" // responses: // '200': // description: GenericResp @@ -137,7 +138,8 @@ func (ctl *DMSController) CreateEnvironmentTag(c echo.Context) error { // description: the name of environment tag to be updated // required: true // in: body -// type: string +// schema: +// "$ref": "#/definitions/UpdateEnvironmentTagReq" // responses: // '200': // description: GenericResp @@ -2713,6 +2715,10 @@ func (ctl *DMSController) GetOauth2Tips(c echo.Context) error { // swagger:route GET /v1/dms/oauth2/link OAuth2 Oauth2LinkOrCallback // // Oauth2 Link or Callback. +// +// responses: +// 302: description: Redirect to identity provider or post-login callback URL +// default: body:GenericResp func (ctl *DMSController) Oauth2LinkOrCallback(c echo.Context) error { // 如果是OAuth2 callback请求，QueryParam中会有code这个参数 if c.QueryParam("code") == "" { @@ -3391,14 +3397,8 @@ func (ctl *DMSController) SendSmsCode(context echo.Context) error { // // --- // parameters: -// - name: code -// description: verify sms code -// required: true -// in: body -// schema: -// "$ref": "#/definitions/VerifySmsCodeReq" -// - name: username -// description: user name +// - name: body +// description: verify sms code request // required: true // in: body // schema: @@ -3856,13 +3856,53 @@ func (ctl *DMSController) GetGlobalDataExportWorkflows(c echo.Context) error { return NewOkRespWithReply(c, reply) } -// swagger:route GET /v1/dms/projects/data_export_workflows DataExportWorkflows ListAllDataExportWorkflows +// swagger:operation GET /v1/dms/projects/data_export_workflows DataExportWorkflows ListAllDataExportWorkflows // // List all data_export workflow. // -// responses: -// 200: body:ListDataExportWorkflowsReply -// default: body:GenericResp +// --- +// parameters: +// - name: page_size +// in: query +// required: true +// type: integer +// format: uint32 +// - name: page_index +// in: query +// type: integer +// format: uint32 +// - name: filter_by_status +// in: query +// type: string +// - name: filter_by_create_user_uid +// in: query +// type: string +// - name: filter_current_step_assignee_user_uid +// in: query +// type: string +// - name: filter_by_db_service_uid +// in: query +// type: string +// - name: filter_create_time_from +// in: query +// type: string +// - name: filter_create_time_to +// in: query +// type: string +// - name: fuzzy_keyword +// in: query +// type: string +// +// responses: +// +// '200': +// description: ListDataExportWorkflowsReply +// schema: +// "$ref": "#/definitions/ListDataExportWorkflowsReply" +// default: +// description: GenericResp +// schema: +// "$ref": "#/definitions/GenericResp" func (ctl *DMSController) ListAllDataExportWorkflows(c echo.Context) error { req := new(aV1.ListDataExportWorkflowsReq) err := bindAndValidateReq(c, req) From 82cb72465aae4aa34323c28d9c3aed5043d1ebfa Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Thu, 14 May 2026 12:32:30 +0000 Subject: [PATCH 19/27] fix swagger --- api/dms/service/v1/data_export_workflow.go | 2 +- api/dms/service/v1/db_structure_columns.go | 3 - api/dms/service/v1/masking.go | 5 +- api/dms/service/v1/unmasking_workflow.go | 7 -- .../apiserver/service/data_mask_controller.go | 6 -- internal/apiserver/service/dms_controller.go | 68 ++++--------------- 6 files changed, 18 insertions(+), 73 deletions(-) diff --git a/api/dms/service/v1/data_export_workflow.go b/api/dms/service/v1/data_export_workflow.go index 0273ec6f..c2721ab5 100644 --- a/api/dms/service/v1/data_export_workflow.go +++ b/api/dms/service/v1/data_export_workflow.go @@ -41,7 +41,7 @@ type AddDataExportWorkflowReply struct { base.GenericResp } -// swagger:parameters ListDataExportWorkflows +// swagger:parameters ListDataExportWorkflows ListAllDataExportWorkflows type ListDataExportWorkflowsReq struct { // project id // Required: false diff --git a/api/dms/service/v1/db_structure_columns.go b/api/dms/service/v1/db_structure_columns.go index 632e694d..29ee0fff 100644 --- a/api/dms/service/v1/db_structure_columns.go +++ b/api/dms/service/v1/db_structure_columns.go @@ -4,7 +4,6 @@ import ( base "github.com/actiontech/dms/pkg/dms-common/api/base/v1" ) -// swagger:parameters ListTableColumns type ListTableColumnsReq struct { // Required: true // in:path @@ -20,14 +19,12 @@ type ListTableColumnsReq struct { TableName string `param:"table_name" json:"table_name" validate:"required"` } -// swagger:model ListTableColumnsReply type ListTableColumnsReply struct { Data []*TableColumn `json:"data"` // Generic reply base.GenericResp } -// swagger:model TableColumn type TableColumn struct { Name string `json:"name"` Type string `json:"type"` diff --git a/api/dms/service/v1/masking.go b/api/dms/service/v1/masking.go index d991780e..94f60bb8 100644 --- a/api/dms/service/v1/masking.go +++ b/api/dms/service/v1/masking.go @@ -8,10 +8,11 @@ import ( // swagger:parameters ListMaskingRules type ListMaskingRulesReq struct { - // project uid（项目路径下由 path 注入；全局 GET /v1/dms/masking/rules 可不传，仅返回内置规则） + // project uid // in: path + // Required: true // Example: "project_uid" - ProjectUid string `param:"project_uid" query:"project_uid" json:"project_uid"` + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` // 规则来源筛选: builtin 或 custom，为空时返回全部 // in: query // Example: "custom" diff --git a/api/dms/service/v1/unmasking_workflow.go b/api/dms/service/v1/unmasking_workflow.go index baff7685..633f915b 100644 --- a/api/dms/service/v1/unmasking_workflow.go +++ b/api/dms/service/v1/unmasking_workflow.go @@ -154,11 +154,6 @@ type UnmaskingSQLDetail struct { // 血缘分析快照 LineageAnalysisSnapshot *biz.AnalyzeResult `json:"lineage_analysis_snapshot,omitempty"` - UnmaskingSQLPreviewData -} - -// swagger:model UnmaskingSQLPreviewData -type UnmaskingSQLPreviewData struct { // 脱敏后的预览数据 (普通用户仅能看到此数据) MaskedData *SQLQueryResult `json:"masked_data"` // 原始采样数据 (仅有权限的审核人能看到) @@ -193,7 +188,6 @@ type UnmaskingOperationLogItem struct { ExtraMessage string `json:"extra_message"` } -// swagger:model ApproveUnmaskingWorkflowReq type ApproveUnmaskingWorkflowReq struct { // swagger:ignore ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` @@ -217,7 +211,6 @@ type ApproveUnmaskingWorkflowReply struct { base.GenericResp } -// swagger:model RejectUnmaskingWorkflowReq type RejectUnmaskingWorkflowReq struct { // swagger:ignore ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` diff --git a/internal/apiserver/service/data_mask_controller.go b/internal/apiserver/service/data_mask_controller.go index 0c5fdac2..fc032186 100644 --- a/internal/apiserver/service/data_mask_controller.go +++ b/internal/apiserver/service/data_mask_controller.go @@ -51,13 +51,7 @@ func (ctl *DMSController) ListMaskingTemplates(c echo.Context) error { return NewOkRespWithReply(c, reply) } -// swagger:route GET /v1/dms/projects/{project_uid}/db_services/{db_service_uid}/schemas/{schema_name}/tables/{table_name}/columns DBStructure ListTableColumns -// // List table columns (internal API for lineage analysis). -// -// responses: -// 200: body:ListTableColumnsReply -// default: body:GenericResp func (ctl *DMSController) ListTableColumns(c echo.Context) error { // 内部接口，仅允许sys/admin用户访问 currentUserUid, err := jwt.GetUserUidStrFromContext(c) diff --git a/internal/apiserver/service/dms_controller.go b/internal/apiserver/service/dms_controller.go index 17b9108d..5582b58e 100644 --- a/internal/apiserver/service/dms_controller.go +++ b/internal/apiserver/service/dms_controller.go @@ -86,8 +86,7 @@ func (ctl *DMSController) Shutdown() error { // description: the name of environment tag to be created // in: body // required: true -// schema: -// "$ref": "#/definitions/CreateEnvironmentTagReq" +// type: string // responses: // '200': // description: GenericResp @@ -138,8 +137,7 @@ func (ctl *DMSController) CreateEnvironmentTag(c echo.Context) error { // description: the name of environment tag to be updated // required: true // in: body -// schema: -// "$ref": "#/definitions/UpdateEnvironmentTagReq" +// type: string // responses: // '200': // description: GenericResp @@ -2715,10 +2713,6 @@ func (ctl *DMSController) GetOauth2Tips(c echo.Context) error { // swagger:route GET /v1/dms/oauth2/link OAuth2 Oauth2LinkOrCallback // // Oauth2 Link or Callback. -// -// responses: -// 302: description: Redirect to identity provider or post-login callback URL -// default: body:GenericResp func (ctl *DMSController) Oauth2LinkOrCallback(c echo.Context) error { // 如果是OAuth2 callback请求，QueryParam中会有code这个参数 if c.QueryParam("code") == "" { @@ -3397,8 +3391,14 @@ func (ctl *DMSController) SendSmsCode(context echo.Context) error { // // --- // parameters: -// - name: body -// description: verify sms code request +// - name: code +// description: verify sms code +// required: true +// in: body +// schema: +// "$ref": "#/definitions/VerifySmsCodeReq" +// - name: username +// description: user name // required: true // in: body // schema: @@ -3856,53 +3856,13 @@ func (ctl *DMSController) GetGlobalDataExportWorkflows(c echo.Context) error { return NewOkRespWithReply(c, reply) } -// swagger:operation GET /v1/dms/projects/data_export_workflows DataExportWorkflows ListAllDataExportWorkflows +// swagger:route GET /v1/dms/projects/data_export_workflows DataExportWorkflows ListAllDataExportWorkflows // // List all data_export workflow. // -// --- -// parameters: -// - name: page_size -// in: query -// required: true -// type: integer -// format: uint32 -// - name: page_index -// in: query -// type: integer -// format: uint32 -// - name: filter_by_status -// in: query -// type: string -// - name: filter_by_create_user_uid -// in: query -// type: string -// - name: filter_current_step_assignee_user_uid -// in: query -// type: string -// - name: filter_by_db_service_uid -// in: query -// type: string -// - name: filter_create_time_from -// in: query -// type: string -// - name: filter_create_time_to -// in: query -// type: string -// - name: fuzzy_keyword -// in: query -// type: string -// -// responses: -// -// '200': -// description: ListDataExportWorkflowsReply -// schema: -// "$ref": "#/definitions/ListDataExportWorkflowsReply" -// default: -// description: GenericResp -// schema: -// "$ref": "#/definitions/GenericResp" +// responses: +// 200: body:ListDataExportWorkflowsReply +// default: body:GenericResp func (ctl *DMSController) ListAllDataExportWorkflows(c echo.Context) error { req := new(aV1.ListDataExportWorkflowsReq) err := bindAndValidateReq(c, req) From 0971ce5f10babc69f1b63f9750d4ebe3f77e2826 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Thu, 14 May 2026 12:01:10 +0000 Subject: [PATCH 20/27] fix(api): improve error handling in APIServer response Refactored error handling in the APIServer to dynamically set the HTTP status code and response code based on the type of error encountered. This change enhances the clarity of error responses by ensuring the correct status code is returned, improving the overall API error communication. --- internal/apiserver/service/service.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/apiserver/service/service.go b/internal/apiserver/service/service.go index b507b425..cdcc4d53 100644 --- a/internal/apiserver/service/service.go +++ b/internal/apiserver/service/service.go @@ -39,7 +39,7 @@ import ( ) type APIServer struct { - DMSController *DMSController + DMSController *DMSController SqlWorkbenchController *SqlWorkbenchController // more controllers @@ -58,14 +58,18 @@ func NewAPIServer(logger utilLog.Logger, opts *conf.DMSOptions) (*APIServer, err } message := err.Error() + statusCode := http.StatusBadRequest + respCode := int(apiError.BadRequestErr) if httpErr, ok := err.(*echo.HTTPError); ok { + statusCode = httpErr.Code + respCode = httpErr.Code if msg, ok := httpErr.Message.(string); ok && msg != "" { message = msg } } - _ = c.JSON(http.StatusBadRequest, bV1.GenericResp{ - Code: int(apiError.BadRequestErr), + _ = c.JSON(statusCode, bV1.GenericResp{ + Code: respCode, Message: message, }) } From ea14f4ff062ccadba013a4ef699291e5067ceb86 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Thu, 14 May 2026 13:27:08 +0000 Subject: [PATCH 21/27] gen swag --- api/swagger.json | 1382 +++++++++++++++++++++++++++++++++++++++------- api/swagger.yaml | 1063 +++++++++++++++++++++++++++++++---- 2 files changed, 2142 insertions(+), 303 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index e0e7b566..725c0525 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -4353,150 +4353,6 @@ } } }, - "/v1/dms/projects/{project_uid}/masking/approval-requests/pending": { - "get": { - "tags": [ - "Masking" - ], - "summary": "查询待审批申请列表。", - "operationId": "ListPendingApprovalRequests", - "parameters": [ - { - "type": "string", - "example": "\"project_uid\"", - "x-go-name": "ProjectUid", - "description": "project uid", - "name": "project_uid", - "in": "path", - "required": true - }, - { - "type": "integer", - "format": "uint32", - "example": "20", - "x-go-name": "PageSize", - "description": "the maximum count of requests to be returned, default is 20", - "name": "page_size", - "in": "query" - }, - { - "type": "integer", - "format": "uint32", - "example": "0", - "x-go-name": "PageIndex", - "description": "the offset of requests to be returned, default is 0", - "name": "page_index", - "in": "query" - } - ], - "responses": { - "200": { - "description": "ListPendingApprovalRequestsReply", - "schema": { - "$ref": "#/definitions/ListPendingApprovalRequestsReply" - } - }, - "default": { - "description": "GenericResp", - "schema": { - "$ref": "#/definitions/GenericResp" - } - } - } - } - }, - "/v1/dms/projects/{project_uid}/masking/approval-requests/{request_id}": { - "get": { - "tags": [ - "Masking" - ], - "summary": "获取明文访问申请详情。", - "operationId": "GetPlaintextAccessRequestDetail", - "parameters": [ - { - "type": "string", - "example": "\"project_uid\"", - "x-go-name": "ProjectUid", - "description": "project uid", - "name": "project_uid", - "in": "path", - "required": true - }, - { - "type": "integer", - "format": "int64", - "example": 1, - "x-go-name": "RequestID", - "description": "approval request id", - "name": "request_id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "GetPlaintextAccessRequestDetailReply", - "schema": { - "$ref": "#/definitions/GetPlaintextAccessRequestDetailReply" - } - }, - "default": { - "description": "GenericResp", - "schema": { - "$ref": "#/definitions/GenericResp" - } - } - } - } - }, - "/v1/dms/projects/{project_uid}/masking/approval-requests/{request_id}/decisions": { - "post": { - "tags": [ - "Masking" - ], - "summary": "处理审批申请。", - "operationId": "ProcessApprovalRequest", - "parameters": [ - { - "type": "string", - "description": "项目 UID", - "name": "project_uid", - "in": "path", - "required": true - }, - { - "type": "integer", - "description": "审批申请 ID", - "name": "request_id", - "in": "path", - "required": true - }, - { - "description": "处理动作信息", - "name": "action", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/ProcessApprovalRequestReq" - } - } - ], - "responses": { - "200": { - "description": "成功处理审批申请", - "schema": { - "$ref": "#/definitions/ProcessApprovalRequestReply" - } - }, - "default": { - "description": "通用错误响应", - "schema": { - "$ref": "#/definitions/GenericResp" - } - } - } - } - }, "/v1/dms/projects/{project_uid}/masking/overview": { "get": { "tags": [ @@ -5713,6 +5569,305 @@ } } }, + "/v1/dms/projects/{project_uid}/masking/unmasking-workflows": { + "get": { + "tags": [ + "Masking" + ], + "summary": "List unmasking workflows.", + "operationId": "ListUnmaskingWorkflows", + "parameters": [ + { + "type": "string", + "x-go-name": "ProjectUid", + "description": "project id", + "name": "project_uid", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "uint32", + "x-go-name": "PageSize", + "description": "the maximum count of workflows to be returned", + "name": "page_size", + "in": "query", + "required": true + }, + { + "type": "integer", + "format": "uint32", + "x-go-name": "PageIndex", + "description": "the offset of workflows to be returned, default is 0", + "name": "page_index", + "in": "query" + }, + { + "enum": [ + "pending", + "approved", + "rejected", + "cancelled" + ], + "type": "string", + "x-go-enum-desc": "pending UnmaskingWorkflowApprovalStatusPending 待审批\napproved UnmaskingWorkflowApprovalStatusApproved 已批准\nrejected UnmaskingWorkflowApprovalStatusRejected 已驳回\ncancelled UnmaskingWorkflowApprovalStatusCancelled 已取消", + "x-go-name": "FilterByApprovalStatus", + "description": "filter the approval status", + "name": "filter_by_approval_status", + "in": "query" + }, + { + "enum": [ + "unviewed", + "viewed" + ], + "type": "string", + "x-go-enum-desc": "unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看\nviewed UnmaskingWorkflowUsageStatusViewed 已查看", + "x-go-name": "FilterByUsageStatus", + "description": "filter the usage status", + "name": "filter_by_usage_status", + "in": "query" + }, + { + "type": "string", + "x-go-name": "FilterByDBServiceUid", + "description": "filter db_service id", + "name": "filter_by_db_service_uid", + "in": "query" + } + ], + "responses": { + "200": { + "description": "List unmasking workflows successfully", + "schema": { + "$ref": "#/definitions/ListUnmaskingWorkflowsReply" + } + }, + "default": { + "description": "Generic error response", + "schema": { + "$ref": "#/definitions/GenericResp" + } + } + } + }, + "post": { + "tags": [ + "Masking" + ], + "summary": "Create unmasking workflow.", + "operationId": "CreateUnmaskingWorkflow", + "parameters": [ + { + "type": "string", + "description": "project id", + "name": "project_uid", + "in": "path", + "required": true + }, + { + "description": "unmasking workflow info", + "name": "unmasking_workflow", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/CreateUnmaskingWorkflowReq" + } + } + ], + "responses": { + "200": { + "description": "Create unmasking workflow successfully", + "schema": { + "$ref": "#/definitions/CreateUnmaskingWorkflowReply" + } + }, + "default": { + "description": "Generic error response", + "schema": { + "$ref": "#/definitions/GenericResp" + } + } + } + } + }, + "/v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}": { + "get": { + "tags": [ + "Masking" + ], + "summary": "Get unmasking workflow detail.", + "operationId": "GetUnmaskingWorkflow", + "parameters": [ + { + "type": "string", + "x-go-name": "ProjectUid", + "description": "project id", + "name": "project_uid", + "in": "path", + "required": true + }, + { + "type": "string", + "x-go-name": "WorkflowID", + "description": "workflow id", + "name": "workflow_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Get unmasking workflow detail successfully", + "schema": { + "$ref": "#/definitions/GetUnmaskingWorkflowReply" + } + }, + "default": { + "description": "Generic error response", + "schema": { + "$ref": "#/definitions/GenericResp" + } + } + } + } + }, + "/v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/approve": { + "post": { + "tags": [ + "Masking" + ], + "summary": "Approve unmasking workflow.", + "operationId": "ApproveUnmaskingWorkflow", + "parameters": [ + { + "type": "string", + "description": "project id", + "name": "project_uid", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "workflow id", + "name": "workflow_id", + "in": "path", + "required": true + }, + { + "description": "approve unmasking workflow info (optional, only carries approve_reason)", + "name": "approve_unmasking_workflow", + "in": "body", + "schema": { + "$ref": "#/definitions/ApproveUnmaskingWorkflow" + } + } + ], + "responses": { + "200": { + "description": "Approve unmasking workflow successfully", + "schema": { + "$ref": "#/definitions/ApproveUnmaskingWorkflowReply" + } + }, + "default": { + "description": "Generic error response", + "schema": { + "$ref": "#/definitions/GenericResp" + } + } + } + } + }, + "/v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/cancel": { + "post": { + "tags": [ + "Masking" + ], + "summary": "Cancel unmasking workflow.", + "operationId": "CancelUnmaskingWorkflow", + "parameters": [ + { + "type": "string", + "x-go-name": "ProjectUid", + "description": "project id", + "name": "project_uid", + "in": "path", + "required": true + }, + { + "type": "string", + "x-go-name": "WorkflowID", + "description": "workflow id", + "name": "workflow_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Cancel unmasking workflow successfully", + "schema": { + "$ref": "#/definitions/CancelUnmaskingWorkflowReply" + } + }, + "default": { + "description": "Generic error response", + "schema": { + "$ref": "#/definitions/GenericResp" + } + } + } + } + }, + "/v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/reject": { + "post": { + "tags": [ + "Masking" + ], + "summary": "Reject unmasking workflow.", + "operationId": "RejectUnmaskingWorkflow", + "parameters": [ + { + "type": "string", + "description": "project id", + "name": "project_uid", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "workflow id", + "name": "workflow_id", + "in": "path", + "required": true + }, + { + "description": "reject unmasking workflow info", + "name": "reject_unmasking_workflow", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/RejectUnmaskingWorkflow" + } + } + ], + "responses": { + "200": { + "description": "Reject unmasking workflow successfully", + "schema": { + "$ref": "#/definitions/RejectUnmaskingWorkflowReply" + } + }, + "default": { + "description": "Generic error response", + "schema": { + "$ref": "#/definitions/GenericResp" + } + } + } + } + }, "/v1/dms/projects/{project_uid}/member_groups": { "get": { "tags": [ @@ -9051,21 +9206,156 @@ "message": { "description": "message", "type": "string", - "x-go-name": "Message" + "x-go-name": "Message" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "AddUserGroupReq": { + "type": "object", + "properties": { + "user_group": { + "$ref": "#/definitions/UserGroup" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "AddUserReply": { + "type": "object", + "properties": { + "code": { + "description": "code", + "type": "integer", + "format": "int64", + "x-go-name": "Code" + }, + "data": { + "description": "Add user reply", + "type": "object", + "properties": { + "uid": { + "description": "user UID", + "type": "string", + "x-go-name": "Uid" + } + }, + "x-go-name": "Data" + }, + "message": { + "description": "message", + "type": "string", + "x-go-name": "Message" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "AddUserReq": { + "type": "object", + "properties": { + "user": { + "$ref": "#/definitions/User" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "AdditionalParam": { + "type": "object", + "properties": { + "description": { + "type": "string", + "x-go-name": "Description" + }, + "name": { + "type": "string", + "x-go-name": "Name" + }, + "type": { + "type": "string", + "x-go-name": "Type" + }, + "value": { + "type": "string", + "x-go-name": "Value" + } + }, + "x-go-package": "github.com/actiontech/dms/pkg/dms-common/api/dms/v1" + }, + "AnalyzeResult": { + "type": "object", + "properties": { + "edges": { + "description": "血缘图边", + "type": "array", + "items": { + "$ref": "#/definitions/LineageEdge" + }, + "x-go-name": "Edges" + }, + "nodes": { + "description": "血缘图节点", + "type": "array", + "items": { + "$ref": "#/definitions/LineageNode" + }, + "x-go-name": "Nodes" + }, + "original_sql": { + "description": "原始 SQL", + "type": "string", + "x-go-name": "OriginalSQL" + }, + "result_columns": { + "description": "解析到的结果列列表", + "type": "array", + "items": { + "$ref": "#/definitions/ResultColumn" + }, + "x-go-name": "ResultColumns" + }, + "source_columns": { + "description": "解析到的源列列表", + "type": "array", + "items": { + "$ref": "#/definitions/ColumnRef" + }, + "x-go-name": "SourceColumns" + }, + "tables": { + "description": "解析到的表引用列表", + "type": "array", + "items": { + "$ref": "#/definitions/TableRef" + }, + "x-go-name": "Tables" + }, + "title": { + "description": "分析标题/摘要", + "type": "string", + "x-go-name": "Title" + }, + "warnings": { + "description": "警告信息（解析不完整等）", + "type": "array", + "items": { + "type": "string" + }, + "x-go-name": "Warnings" } }, - "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + "x-go-package": "github.com/actiontech/dms/internal/data_masking/biz" }, - "AddUserGroupReq": { + "ApproveUnmaskingWorkflow": { "type": "object", "properties": { - "user_group": { - "$ref": "#/definitions/UserGroup" + "approve_reason": { + "description": "审批理由 非必须", + "type": "string", + "x-go-name": "ApproveReason" } }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, - "AddUserReply": { + "ApproveUnmaskingWorkflowReply": { "type": "object", "properties": { "code": { @@ -9074,18 +9364,6 @@ "format": "int64", "x-go-name": "Code" }, - "data": { - "description": "Add user reply", - "type": "object", - "properties": { - "uid": { - "description": "user UID", - "type": "string", - "x-go-name": "Uid" - } - }, - "x-go-name": "Data" - }, "message": { "description": "message", "type": "string", @@ -9094,37 +9372,6 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, - "AddUserReq": { - "type": "object", - "properties": { - "user": { - "$ref": "#/definitions/User" - } - }, - "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" - }, - "AdditionalParam": { - "type": "object", - "properties": { - "description": { - "type": "string", - "x-go-name": "Description" - }, - "name": { - "type": "string", - "x-go-name": "Name" - }, - "type": { - "type": "string", - "x-go-name": "Type" - }, - "value": { - "type": "string", - "x-go-name": "Value" - } - }, - "x-go-package": "github.com/actiontech/dms/pkg/dms-common/api/dms/v1" - }, "AuditPlanTypes": { "type": "object", "properties": { @@ -9450,6 +9697,23 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "CancelUnmaskingWorkflowReply": { + "type": "object", + "properties": { + "code": { + "description": "code", + "type": "integer", + "format": "int64", + "x-go-name": "Code" + }, + "message": { + "description": "message", + "type": "string", + "x-go-name": "Message" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, "CheckDBServiceIsConnectableByIdReq": { "type": "object", "properties": { @@ -9738,6 +10002,116 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "ColumnMaskingConfig": { + "description": "ColumnMaskingConfig 列级别脱敏配置领域模型", + "type": "object", + "properties": { + "column_id": { + "description": "列 ID（db_columns.id）", + "type": "integer", + "format": "uint64", + "x-go-name": "ColumnID" + }, + "column_name": { + "description": "列名", + "type": "string", + "x-go-name": "ColumnName" + }, + "confidence": { + "description": "置信度\nHigh ConfidenceHigh 高：高度确信为敏感数据\nMedium ConfidenceMedium 中：中等确信为敏感数据\nLow ConfidenceLow 低：低确信为敏感数据", + "type": "string", + "enum": [ + "High", + "Medium", + "Low" + ], + "x-go-enum-desc": "High ConfidenceHigh 高：高度确信为敏感数据\nMedium ConfidenceMedium 中：中等确信为敏感数据\nLow ConfidenceLow 低：低确信为敏感数据", + "x-go-name": "Confidence" + }, + "created_at": { + "description": "创建时间", + "type": "string", + "format": "date-time", + "x-go-name": "CreatedAt" + }, + "db_service_uid": { + "description": "数据源 UID", + "type": "string", + "x-go-name": "DBServiceUID" + }, + "id": { + "description": "配置记录 ID", + "type": "integer", + "format": "uint64", + "x-go-name": "ID" + }, + "is_masking_enabled": { + "description": "是否启用脱敏", + "type": "boolean", + "x-go-name": "IsMaskingEnabled" + }, + "masking_rule_id": { + "description": "脱敏规则 ID", + "type": "integer", + "format": "int64", + "x-go-name": "MaskingRuleID" + }, + "masking_rule_name": { + "description": "脱敏规则名称（中文）", + "type": "string", + "x-go-name": "MaskingRuleName" + }, + "schema_name": { + "description": "Schema 名称", + "type": "string", + "x-go-name": "SchemaName" + }, + "status": { + "description": "配置状态\nPENDING_CONFIRM MaskingConfigStatusPendingConfirm\nCONFIGURED MaskingConfigStatusConfigured\nSYSTEM_CONFIRMED MaskingConfigStatusSystemConfirmed", + "type": "string", + "enum": [ + "PENDING_CONFIRM", + "CONFIGURED", + "SYSTEM_CONFIRMED" + ], + "x-go-enum-desc": "PENDING_CONFIRM MaskingConfigStatusPendingConfirm\nCONFIGURED MaskingConfigStatusConfigured\nSYSTEM_CONFIRMED MaskingConfigStatusSystemConfirmed", + "x-go-name": "Status" + }, + "table_name": { + "description": "表名", + "type": "string", + "x-go-name": "TableName" + }, + "updated_at": { + "description": "更新时间", + "type": "string", + "format": "date-time", + "x-go-name": "UpdatedAt" + } + }, + "x-go-package": "github.com/actiontech/dms/internal/data_masking/biz" + }, + "ColumnRef": { + "type": "object", + "properties": { + "column": { + "description": "列名", + "type": "string", + "x-go-name": "Column" + }, + "schema": { + "description": "Schema 名称", + "type": "string", + "x-go-name": "Schema" + }, + "table": { + "description": "表名", + "type": "string", + "x-go-name": "Table" + } + }, + "x-go-package": "github.com/actiontech/dms/internal/data_masking/biz" + }, "CompanyNotice": { "description": "A companynotice", "type": "object", @@ -9840,6 +10214,108 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "CreateUnmaskingSQLItem": { + "type": "object", + "properties": { + "sql_content": { + "description": "原始 SQL 内容", + "type": "string", + "x-go-name": "SQLContent" + }, + "sql_index_id": { + "description": "来源侧 SQL 索引 id（如数据导出记录中的 SQL 序号）；与 SQL 工作台场景的索引约定可能不同，需结合 source_type、source_uid 解析", + "type": "string", + "x-go-name": "SQLIndexID" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "CreateUnmaskingWorkflow": { + "type": "object", + "properties": { + "apply_reason": { + "description": "申请理由", + "type": "string", + "x-go-name": "ApplyReason" + }, + "datasource_uid": { + "description": "数据源 UID", + "type": "string", + "x-go-name": "DatasourceUID" + }, + "default_schema": { + "description": "SQL 默认 schema", + "type": "string", + "x-go-name": "DefaultSchema" + }, + "source_type": { + "description": "来源类型\ndata_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单\nsql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台", + "type": "string", + "enum": [ + "data_export", + "sql_workbench" + ], + "x-go-enum-desc": "data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单\nsql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台", + "x-go-name": "SourceType" + }, + "source_uid": { + "description": "来源对象 UID (如数据导出任务 UID)", + "type": "string", + "x-go-name": "SourceUID" + }, + "unmasking_sqls": { + "description": "待脱敏 SQL 列表", + "type": "array", + "items": { + "$ref": "#/definitions/CreateUnmaskingSQLItem" + }, + "x-go-name": "UnmaskingSQLs" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "CreateUnmaskingWorkflowReply": { + "type": "object", + "properties": { + "code": { + "description": "code", + "type": "integer", + "format": "int64", + "x-go-name": "Code" + }, + "data": { + "$ref": "#/definitions/CreateUnmaskingWorkflowReplyData" + }, + "message": { + "description": "message", + "type": "string", + "x-go-name": "Message" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "CreateUnmaskingWorkflowReplyData": { + "type": "object", + "properties": { + "workflow_id": { + "type": "string", + "x-go-name": "WorkflowID" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "CreateUnmaskingWorkflowReq": { + "type": "object", + "required": [ + "unmasking_workflow" + ], + "properties": { + "unmasking_workflow": { + "$ref": "#/definitions/CreateUnmaskingWorkflow" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, "CurrentProjectAdmin": { "type": "object", "properties": { @@ -11608,6 +12084,26 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "GetUnmaskingWorkflowReply": { + "type": "object", + "properties": { + "code": { + "description": "code", + "type": "integer", + "format": "int64", + "x-go-name": "Code" + }, + "data": { + "$ref": "#/definitions/UnmaskingWorkflowDetail" + }, + "message": { + "description": "message", + "type": "string", + "x-go-name": "Message" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, "GetUser": { "description": "A dms user", "type": "object", @@ -12421,7 +12917,82 @@ "x-go-name": "Used" } }, - "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "LineageEdge": { + "type": "object", + "properties": { + "from_id": { + "description": "起点节点 ID", + "type": "string", + "x-go-name": "FromID" + }, + "to_id": { + "description": "终点节点 ID", + "type": "string", + "x-go-name": "ToID" + }, + "type": { + "description": "边类型\ndirect EdgeTypeDirect 直接依赖\ntransform EdgeTypeTransform 转换/计算依赖\naggregate EdgeTypeAggregate 聚合依赖", + "type": "string", + "enum": [ + "direct", + "transform", + "aggregate" + ], + "x-go-enum-desc": "direct EdgeTypeDirect 直接依赖\ntransform EdgeTypeTransform 转换/计算依赖\naggregate EdgeTypeAggregate 聚合依赖", + "x-go-name": "Type" + } + }, + "x-go-package": "github.com/actiontech/dms/internal/data_masking/biz" + }, + "LineageNode": { + "type": "object", + "properties": { + "column": { + "description": "列名（列节点可能存在）", + "type": "string", + "x-go-name": "Column" + }, + "expr": { + "description": "表达式内容（表达式节点可能存在）", + "type": "string", + "x-go-name": "Expr" + }, + "id": { + "description": "节点 ID（图内唯一）", + "type": "string", + "x-go-name": "ID" + }, + "name": { + "description": "节点展示名", + "type": "string", + "x-go-name": "Name" + }, + "schema": { + "description": "Schema 名称（列节点可能存在）", + "type": "string", + "x-go-name": "Schema" + }, + "table": { + "description": "表名（列节点可能存在）", + "type": "string", + "x-go-name": "Table" + }, + "type": { + "description": "节点类型\nsource_column NodeTypeSource 源列节点\nexpression NodeTypeExpression 表达式节点\nresult_column NodeTypeResult 结果列节点\ntable NodeTypeTable 表节点", + "type": "string", + "enum": [ + "source_column", + "expression", + "result_column", + "table" + ], + "x-go-enum-desc": "source_column NodeTypeSource 源列节点\nexpression NodeTypeExpression 表达式节点\nresult_column NodeTypeResult 结果列节点\ntable NodeTypeTable 表节点", + "x-go-name": "Type" + } + }, + "x-go-package": "github.com/actiontech/dms/internal/data_masking/biz" }, "ListBusinessTagsReply": { "type": "object", @@ -13159,6 +13730,17 @@ "type": "string", "x-go-name": "ExportSQLType" }, + "lineage_analysis_snapshot": { + "$ref": "#/definitions/AnalyzeResult" + }, + "masking_config_snapshot": { + "description": "脱敏配置快照", + "type": "array", + "items": { + "$ref": "#/definitions/ColumnMaskingConfig" + }, + "x-go-name": "MaskingConfigSnapshot" + }, "sql": { "type": "string", "x-go-name": "ExportSQL" @@ -14829,6 +15411,35 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "ListUnmaskingWorkflowsReply": { + "type": "object", + "properties": { + "code": { + "description": "code", + "type": "integer", + "format": "int64", + "x-go-name": "Code" + }, + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/UnmaskingWorkflowListItem" + }, + "x-go-name": "Data" + }, + "message": { + "description": "message", + "type": "string", + "x-go-name": "Message" + }, + "total_nums": { + "type": "integer", + "format": "int64", + "x-go-name": "Total" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, "ListUser": { "description": "A dms user", "type": "object", @@ -16444,6 +17055,37 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "RejectUnmaskingWorkflow": { + "type": "object", + "required": [ + "reject_reason" + ], + "properties": { + "reject_reason": { + "description": "驳回理由", + "type": "string", + "x-go-name": "RejectReason" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "RejectUnmaskingWorkflowReply": { + "type": "object", + "properties": { + "code": { + "description": "code", + "type": "integer", + "format": "int64", + "x-go-name": "Code" + }, + "message": { + "description": "message", + "type": "string", + "x-go-name": "Message" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, "ResourceBusiness": { "type": "object", "properties": { @@ -16694,6 +17336,30 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "ResultColumn": { + "type": "object", + "properties": { + "expression": { + "description": "结果列表达式（SQL 片段）", + "type": "string", + "x-go-name": "Expression" + }, + "name": { + "description": "结果列名", + "type": "string", + "x-go-name": "Name" + }, + "sources": { + "description": "来源列列表", + "type": "array", + "items": { + "$ref": "#/definitions/ColumnRef" + }, + "x-go-name": "Sources" + } + }, + "x-go-package": "github.com/actiontech/dms/internal/data_masking/biz" + }, "Role": { "description": "A role", "type": "object", @@ -16832,6 +17498,41 @@ }, "x-go-package": "github.com/actiontech/dms/pkg/dms-common/api/dms/v1" }, + "SQLQueryResult": { + "type": "object", + "properties": { + "columns": { + "description": "列名列表", + "type": "array", + "items": { + "type": "string" + }, + "x-go-name": "Columns" + }, + "row_count": { + "description": "总行数", + "type": "integer", + "format": "int64", + "x-go-name": "RowCount" + }, + "rows": { + "description": "数据行列表 (每一行的数据顺序与 Columns 一致)", + "type": "array", + "items": { + "$ref": "#/definitions/SQLQueryResultRow" + }, + "x-go-name": "Rows" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "SQLQueryResultRow": { + "type": "array", + "items": { + "type": "string" + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, "SendSmsCodeReply": { "type": "object", "properties": { @@ -17173,6 +17874,27 @@ }, "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" }, + "TableRef": { + "type": "object", + "properties": { + "alias": { + "description": "表别名", + "type": "string", + "x-go-name": "Alias" + }, + "schema": { + "description": "Schema 名称", + "type": "string", + "x-go-name": "Schema" + }, + "table": { + "description": "表名", + "type": "string", + "x-go-name": "Table" + } + }, + "x-go-package": "github.com/actiontech/dms/internal/data_masking/biz" + }, "Task": { "type": "object", "properties": { @@ -17589,6 +18311,278 @@ }, "x-go-package": "github.com/actiontech/dms/pkg/dms-common/api/dms/v1" }, + "UnmaskingOperationLogItem": { + "type": "object", + "properties": { + "action": { + "description": "操作动作\nsubmit UnmaskingActionSubmit 提交申请\napprove UnmaskingActionApprove 批准申请\nreject UnmaskingActionReject 驳回申请\nview_unmasking_workflow_detail UnmaskingActionViewOriginalDataWorkflowDetail 查看工单详情\nview_full_original_data UnmaskingActionViewOriginalData 查看原文\ndownload_full_original_data UnmaskingActionDownloadOriginalData 下载原文\ncancel UnmaskingActionCancel 取消申请", + "type": "string", + "enum": [ + "submit", + "approve", + "reject", + "view_unmasking_workflow_detail", + "view_full_original_data", + "download_full_original_data", + "cancel" + ], + "x-go-enum-desc": "submit UnmaskingActionSubmit 提交申请\napprove UnmaskingActionApprove 批准申请\nreject UnmaskingActionReject 驳回申请\nview_unmasking_workflow_detail UnmaskingActionViewOriginalDataWorkflowDetail 查看工单详情\nview_full_original_data UnmaskingActionViewOriginalData 查看原文\ndownload_full_original_data UnmaskingActionDownloadOriginalData 下载原文\ncancel UnmaskingActionCancel 取消申请", + "x-go-name": "Action" + }, + "action_time": { + "description": "操作时间 (RFC3339)", + "type": "string", + "x-go-name": "ActionTime" + }, + "extra_message": { + "description": "额外信息 (如拦截原因)", + "type": "string", + "x-go-name": "ExtraMessage" + }, + "operator_name": { + "description": "操作人姓名", + "type": "string", + "x-go-name": "OperatorName" + }, + "operator_uid": { + "description": "操作人 UID", + "type": "string", + "x-go-name": "OperatorUID" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "UnmaskingSQLDetail": { + "type": "object", + "properties": { + "lineage_analysis_snapshot": { + "$ref": "#/definitions/AnalyzeResult" + }, + "masked_data": { + "$ref": "#/definitions/SQLQueryResult" + }, + "masking_config_snapshot": { + "description": "脱敏配置快照", + "type": "array", + "items": { + "$ref": "#/definitions/ColumnMaskingConfig" + }, + "x-go-name": "MaskingConfigSnapshot" + }, + "original_data": { + "$ref": "#/definitions/SQLQueryResult" + }, + "sql_content": { + "description": "原始 SQL 内容", + "type": "string", + "x-go-name": "SQLContent" + }, + "sql_index_id": { + "description": "来源侧 SQL 索引 id", + "type": "string", + "x-go-name": "SQLIndexID" + }, + "uid": { + "description": "SQL 详情 UID", + "type": "string", + "x-go-name": "UID" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "UnmaskingWorkflowDetail": { + "type": "object", + "properties": { + "applicant_name": { + "description": "申请人用户名", + "type": "string", + "x-go-name": "ApplicantName" + }, + "apply_reason": { + "description": "申请理由", + "type": "string", + "x-go-name": "ApplyReason" + }, + "approval_status": { + "description": "审批状态\npending UnmaskingWorkflowApprovalStatusPending 待审批\napproved UnmaskingWorkflowApprovalStatusApproved 已批准\nrejected UnmaskingWorkflowApprovalStatusRejected 已驳回\ncancelled UnmaskingWorkflowApprovalStatusCancelled 已取消", + "type": "string", + "enum": [ + "pending", + "approved", + "rejected", + "cancelled" + ], + "x-go-enum-desc": "pending UnmaskingWorkflowApprovalStatusPending 待审批\napproved UnmaskingWorkflowApprovalStatusApproved 已批准\nrejected UnmaskingWorkflowApprovalStatusRejected 已驳回\ncancelled UnmaskingWorkflowApprovalStatusCancelled 已取消", + "x-go-name": "ApprovalStatus" + }, + "created_at": { + "description": "申请时间 (RFC3339)", + "type": "string", + "x-go-name": "CreatedAt" + }, + "current_assignees": { + "description": "当前待处理人", + "type": "array", + "items": { + "$ref": "#/definitions/UidWithName" + }, + "x-go-name": "CurrentAssignees" + }, + "datasource_name": { + "description": "数据源实例名称", + "type": "string", + "x-go-name": "DatasourceName" + }, + "datasource_uid": { + "description": "数据源实例ID", + "type": "string", + "x-go-name": "DatasourceUid" + }, + "expire_time": { + "description": "过期时间 (RFC3339)", + "type": "string", + "x-go-name": "ExpireTime" + }, + "operation_logs": { + "description": "操作日志", + "type": "array", + "items": { + "$ref": "#/definitions/UnmaskingOperationLogItem" + }, + "x-go-name": "OperationLogs" + }, + "reject_reason": { + "description": "驳回理由 (整单驳回时)", + "type": "string", + "x-go-name": "RejectReason" + }, + "source_type": { + "description": "来源类型\ndata_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单\nsql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台", + "type": "string", + "enum": [ + "data_export", + "sql_workbench" + ], + "x-go-enum-desc": "data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单\nsql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台", + "x-go-name": "SourceType" + }, + "source_uid": { + "description": "来源对象UID", + "type": "string", + "x-go-name": "SourceUID" + }, + "unmasking_sqls": { + "description": "SQL 详情列表", + "type": "array", + "items": { + "$ref": "#/definitions/UnmaskingSQLDetail" + }, + "x-go-name": "UnmaskingSQLs" + }, + "usage_status": { + "description": "使用情况\nunviewed UnmaskingWorkflowUsageStatusUnviewed 未查看\nviewed UnmaskingWorkflowUsageStatusViewed 已查看", + "type": "string", + "enum": [ + "unviewed", + "viewed" + ], + "x-go-enum-desc": "unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看\nviewed UnmaskingWorkflowUsageStatusViewed 已查看", + "x-go-name": "UsageStatus" + }, + "workflow_id": { + "description": "申请编号", + "type": "string", + "x-go-name": "WorkflowID" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, + "UnmaskingWorkflowListItem": { + "type": "object", + "properties": { + "applicant_name": { + "description": "申请人用户名", + "type": "string", + "x-go-name": "ApplicantName" + }, + "apply_reason": { + "description": "申请理由", + "type": "string", + "x-go-name": "ApplyReason" + }, + "approval_status": { + "description": "审批状态\npending UnmaskingWorkflowApprovalStatusPending 待审批\napproved UnmaskingWorkflowApprovalStatusApproved 已批准\nrejected UnmaskingWorkflowApprovalStatusRejected 已驳回\ncancelled UnmaskingWorkflowApprovalStatusCancelled 已取消", + "type": "string", + "enum": [ + "pending", + "approved", + "rejected", + "cancelled" + ], + "x-go-enum-desc": "pending UnmaskingWorkflowApprovalStatusPending 待审批\napproved UnmaskingWorkflowApprovalStatusApproved 已批准\nrejected UnmaskingWorkflowApprovalStatusRejected 已驳回\ncancelled UnmaskingWorkflowApprovalStatusCancelled 已取消", + "x-go-name": "ApprovalStatus" + }, + "created_at": { + "description": "申请时间 (RFC3339)", + "type": "string", + "x-go-name": "CreatedAt" + }, + "current_assignees": { + "description": "当前待处理人", + "type": "array", + "items": { + "$ref": "#/definitions/UidWithName" + }, + "x-go-name": "CurrentAssignees" + }, + "datasource_name": { + "description": "数据源实例名称", + "type": "string", + "x-go-name": "DatasourceName" + }, + "datasource_uid": { + "description": "数据源实例ID", + "type": "string", + "x-go-name": "DatasourceUid" + }, + "expire_time": { + "description": "过期时间 (RFC3339)", + "type": "string", + "x-go-name": "ExpireTime" + }, + "source_type": { + "description": "来源类型\ndata_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单\nsql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台", + "type": "string", + "enum": [ + "data_export", + "sql_workbench" + ], + "x-go-enum-desc": "data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单\nsql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台", + "x-go-name": "SourceType" + }, + "source_uid": { + "description": "来源对象UID", + "type": "string", + "x-go-name": "SourceUID" + }, + "usage_status": { + "description": "使用情况\nunviewed UnmaskingWorkflowUsageStatusUnviewed 未查看\nviewed UnmaskingWorkflowUsageStatusViewed 已查看", + "type": "string", + "enum": [ + "unviewed", + "viewed" + ], + "x-go-enum-desc": "unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看\nviewed UnmaskingWorkflowUsageStatusViewed 已查看", + "x-go-name": "UsageStatus" + }, + "workflow_id": { + "description": "申请编号", + "type": "string", + "x-go-name": "WorkflowID" + } + }, + "x-go-package": "github.com/actiontech/dms/api/dms/service/v1" + }, "UpdateBusinessTagReq": { "type": "object", "properties": { diff --git a/api/swagger.yaml b/api/swagger.yaml index 70d66e2b..2444ad1c 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -688,6 +688,75 @@ definitions: x-go-name: Value type: object x-go-package: github.com/actiontech/dms/pkg/dms-common/api/dms/v1 + AnalyzeResult: + properties: + edges: + description: 血缘图边 + items: + $ref: '#/definitions/LineageEdge' + type: array + x-go-name: Edges + nodes: + description: 血缘图节点 + items: + $ref: '#/definitions/LineageNode' + type: array + x-go-name: Nodes + original_sql: + description: 原始 SQL + type: string + x-go-name: OriginalSQL + result_columns: + description: 解析到的结果列列表 + items: + $ref: '#/definitions/ResultColumn' + type: array + x-go-name: ResultColumns + source_columns: + description: 解析到的源列列表 + items: + $ref: '#/definitions/ColumnRef' + type: array + x-go-name: SourceColumns + tables: + description: 解析到的表引用列表 + items: + $ref: '#/definitions/TableRef' + type: array + x-go-name: Tables + title: + description: 分析标题/摘要 + type: string + x-go-name: Title + warnings: + description: 警告信息（解析不完整等） + items: + type: string + type: array + x-go-name: Warnings + type: object + x-go-package: github.com/actiontech/dms/internal/data_masking/biz + ApproveUnmaskingWorkflow: + properties: + approve_reason: + description: 审批理由 非必须 + type: string + x-go-name: ApproveReason + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + ApproveUnmaskingWorkflowReply: + properties: + code: + description: code + format: int64 + type: integer + x-go-name: Code + message: + description: message + type: string + x-go-name: Message + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 AuditPlanTypes: properties: audit_plan_id: @@ -925,6 +994,19 @@ definitions: $ref: '#/definitions/CancelDataExportWorkflowPayload' type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + CancelUnmaskingWorkflowReply: + properties: + code: + description: code + format: int64 + type: integer + x-go-name: Code + message: + description: message + type: string + x-go-name: Message + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 CheckDBServiceIsConnectableByIdReq: properties: db_service_uid: @@ -1143,6 +1225,108 @@ definitions: x-go-name: Message type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + ColumnMaskingConfig: + description: ColumnMaskingConfig 列级别脱敏配置领域模型 + properties: + column_id: + description: 列 ID（db_columns.id） + format: uint64 + type: integer + x-go-name: ColumnID + column_name: + description: 列名 + type: string + x-go-name: ColumnName + confidence: + description: |- + 置信度 + High ConfidenceHigh 高：高度确信为敏感数据 + Medium ConfidenceMedium 中：中等确信为敏感数据 + Low ConfidenceLow 低：低确信为敏感数据 + enum: + - High + - Medium + - Low + type: string + x-go-enum-desc: |- + High ConfidenceHigh 高：高度确信为敏感数据 + Medium ConfidenceMedium 中：中等确信为敏感数据 + Low ConfidenceLow 低：低确信为敏感数据 + x-go-name: Confidence + created_at: + description: 创建时间 + format: date-time + type: string + x-go-name: CreatedAt + db_service_uid: + description: 数据源 UID + type: string + x-go-name: DBServiceUID + id: + description: 配置记录 ID + format: uint64 + type: integer + x-go-name: ID + is_masking_enabled: + description: 是否启用脱敏 + type: boolean + x-go-name: IsMaskingEnabled + masking_rule_id: + description: 脱敏规则 ID + format: int64 + type: integer + x-go-name: MaskingRuleID + masking_rule_name: + description: 脱敏规则名称（中文） + type: string + x-go-name: MaskingRuleName + schema_name: + description: Schema 名称 + type: string + x-go-name: SchemaName + status: + description: |- + 配置状态 + PENDING_CONFIRM MaskingConfigStatusPendingConfirm + CONFIGURED MaskingConfigStatusConfigured + SYSTEM_CONFIRMED MaskingConfigStatusSystemConfirmed + enum: + - PENDING_CONFIRM + - CONFIGURED + - SYSTEM_CONFIRMED + type: string + x-go-enum-desc: |- + PENDING_CONFIRM MaskingConfigStatusPendingConfirm + CONFIGURED MaskingConfigStatusConfigured + SYSTEM_CONFIRMED MaskingConfigStatusSystemConfirmed + x-go-name: Status + table_name: + description: 表名 + type: string + x-go-name: TableName + updated_at: + description: 更新时间 + format: date-time + type: string + x-go-name: UpdatedAt + type: object + x-go-package: github.com/actiontech/dms/internal/data_masking/biz + ColumnRef: + properties: + column: + description: 列名 + type: string + x-go-name: Column + schema: + description: Schema 名称 + type: string + x-go-name: Schema + table: + description: 表名 + type: string + x-go-name: Table + type: object + x-go-package: github.com/actiontech/dms/internal/data_masking/biz CompanyNotice: description: A companynotice properties: @@ -1219,6 +1403,87 @@ definitions: x-go-name: Name type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + CreateUnmaskingSQLItem: + properties: + sql_content: + description: 原始 SQL 内容 + type: string + x-go-name: SQLContent + sql_index_id: + description: 来源侧 SQL 索引 id（如数据导出记录中的 SQL 序号）；与 SQL 工作台场景的索引约定可能不同，需结合 source_type、source_uid 解析 + type: string + x-go-name: SQLIndexID + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + CreateUnmaskingWorkflow: + properties: + apply_reason: + description: 申请理由 + type: string + x-go-name: ApplyReason + datasource_uid: + description: 数据源 UID + type: string + x-go-name: DatasourceUID + default_schema: + description: SQL 默认 schema + type: string + x-go-name: DefaultSchema + source_type: + description: |- + 来源类型 + data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单 + sql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台 + enum: + - data_export + - sql_workbench + type: string + x-go-enum-desc: |- + data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单 + sql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台 + x-go-name: SourceType + source_uid: + description: 来源对象 UID (如数据导出任务 UID) + type: string + x-go-name: SourceUID + unmasking_sqls: + description: 待脱敏 SQL 列表 + items: + $ref: '#/definitions/CreateUnmaskingSQLItem' + type: array + x-go-name: UnmaskingSQLs + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + CreateUnmaskingWorkflowReply: + properties: + code: + description: code + format: int64 + type: integer + x-go-name: Code + data: + $ref: '#/definitions/CreateUnmaskingWorkflowReplyData' + message: + description: message + type: string + x-go-name: Message + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + CreateUnmaskingWorkflowReplyData: + properties: + workflow_id: + type: string + x-go-name: WorkflowID + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + CreateUnmaskingWorkflowReq: + properties: + unmasking_workflow: + $ref: '#/definitions/CreateUnmaskingWorkflow' + required: + - unmasking_workflow + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 CurrentProjectAdmin: properties: is_admin: @@ -2575,6 +2840,21 @@ definitions: x-go-name: Message type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + GetUnmaskingWorkflowReply: + properties: + code: + description: code + format: int64 + type: integer + x-go-name: Code + data: + $ref: '#/definitions/UnmaskingWorkflowDetail' + message: + description: message + type: string + x-go-name: Message + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 GetUser: description: A dms user properties: @@ -3228,6 +3508,81 @@ definitions: x-go-name: Used type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + LineageEdge: + properties: + from_id: + description: 起点节点 ID + type: string + x-go-name: FromID + to_id: + description: 终点节点 ID + type: string + x-go-name: ToID + type: + description: |- + 边类型 + direct EdgeTypeDirect 直接依赖 + transform EdgeTypeTransform 转换/计算依赖 + aggregate EdgeTypeAggregate 聚合依赖 + enum: + - direct + - transform + - aggregate + type: string + x-go-enum-desc: |- + direct EdgeTypeDirect 直接依赖 + transform EdgeTypeTransform 转换/计算依赖 + aggregate EdgeTypeAggregate 聚合依赖 + x-go-name: Type + type: object + x-go-package: github.com/actiontech/dms/internal/data_masking/biz + LineageNode: + properties: + column: + description: 列名（列节点可能存在） + type: string + x-go-name: Column + expr: + description: 表达式内容（表达式节点可能存在） + type: string + x-go-name: Expr + id: + description: 节点 ID（图内唯一） + type: string + x-go-name: ID + name: + description: 节点展示名 + type: string + x-go-name: Name + schema: + description: Schema 名称（列节点可能存在） + type: string + x-go-name: Schema + table: + description: 表名（列节点可能存在） + type: string + x-go-name: Table + type: + description: |- + 节点类型 + source_column NodeTypeSource 源列节点 + expression NodeTypeExpression 表达式节点 + result_column NodeTypeResult 结果列节点 + table NodeTypeTable 表节点 + enum: + - source_column + - expression + - result_column + - table + type: string + x-go-enum-desc: |- + source_column NodeTypeSource 源列节点 + expression NodeTypeExpression 表达式节点 + result_column NodeTypeResult 结果列节点 + table NodeTypeTable 表节点 + x-go-name: Type + type: object + x-go-package: github.com/actiontech/dms/internal/data_masking/biz ListBusinessTagsReply: properties: code: @@ -3811,6 +4166,14 @@ definitions: export_sql_type: type: string x-go-name: ExportSQLType + lineage_analysis_snapshot: + $ref: '#/definitions/AnalyzeResult' + masking_config_snapshot: + description: 脱敏配置快照 + items: + $ref: '#/definitions/ColumnMaskingConfig' + type: array + x-go-name: MaskingConfigSnapshot sql: type: string x-go-name: ExportSQL @@ -5228,6 +5591,28 @@ definitions: x-go-name: Message type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + ListUnmaskingWorkflowsReply: + properties: + code: + description: code + format: int64 + type: integer + x-go-name: Code + data: + items: + $ref: '#/definitions/UnmaskingWorkflowListItem' + type: array + x-go-name: Data + message: + description: message + type: string + x-go-name: Message + total_nums: + format: int64 + type: integer + x-go-name: Total + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 ListUser: description: A dms user properties: @@ -6639,6 +7024,29 @@ definitions: $ref: '#/definitions/RejectDataExportWorkflowPayload' type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + RejectUnmaskingWorkflow: + properties: + reject_reason: + description: 驳回理由 + type: string + x-go-name: RejectReason + required: + - reject_reason + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + RejectUnmaskingWorkflowReply: + properties: + code: + description: code + format: int64 + type: integer + x-go-name: Code + message: + description: message + type: string + x-go-name: Message + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 ResourceBusiness: properties: business_tag: @@ -6829,6 +7237,24 @@ definitions: x-go-name: ResourceType type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + ResultColumn: + properties: + expression: + description: 结果列表达式（SQL 片段） + type: string + x-go-name: Expression + name: + description: 结果列名 + type: string + x-go-name: Name + sources: + description: 来源列列表 + items: + $ref: '#/definitions/ColumnRef' + type: array + x-go-name: Sources + type: object + x-go-package: github.com/actiontech/dms/internal/data_masking/biz Role: description: A role properties: @@ -6937,18 +7363,44 @@ definitions: x-go-name: WorkflowExecEnabled type: object x-go-package: github.com/actiontech/dms/pkg/dms-common/api/dms/v1 - SendSmsCodeReply: + SQLQueryResult: properties: - code: - description: code + columns: + description: 列名列表 + items: + type: string + type: array + x-go-name: Columns + row_count: + description: 总行数 format: int64 type: integer - x-go-name: Code - data: - $ref: '#/definitions/SendSmsCodeReplyData' - message: - description: message - type: string + x-go-name: RowCount + rows: + description: 数据行列表 (每一行的数据顺序与 Columns 一致) + items: + $ref: '#/definitions/SQLQueryResultRow' + type: array + x-go-name: Rows + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + SQLQueryResultRow: + items: + type: string + type: array + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + SendSmsCodeReply: + properties: + code: + description: code + format: int64 + type: integer + x-go-name: Code + data: + $ref: '#/definitions/SendSmsCodeReplyData' + message: + description: message + type: string x-go-name: Message type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 @@ -7236,6 +7688,22 @@ definitions: x-go-name: Status type: object x-go-package: github.com/actiontech/dms/api/dms/service/v1 + TableRef: + properties: + alias: + description: 表别名 + type: string + x-go-name: Alias + schema: + description: Schema 名称 + type: string + x-go-name: Schema + table: + description: 表名 + type: string + x-go-name: Table + type: object + x-go-package: github.com/actiontech/dms/internal/data_masking/biz Task: properties: task_uid: @@ -7540,6 +8008,272 @@ definitions: x-go-name: Uid type: object x-go-package: github.com/actiontech/dms/pkg/dms-common/api/dms/v1 + UnmaskingOperationLogItem: + properties: + action: + description: |- + 操作动作 + submit UnmaskingActionSubmit 提交申请 + approve UnmaskingActionApprove 批准申请 + reject UnmaskingActionReject 驳回申请 + view_unmasking_workflow_detail UnmaskingActionViewOriginalDataWorkflowDetail 查看工单详情 + view_full_original_data UnmaskingActionViewOriginalData 查看原文 + download_full_original_data UnmaskingActionDownloadOriginalData 下载原文 + cancel UnmaskingActionCancel 取消申请 + enum: + - submit + - approve + - reject + - view_unmasking_workflow_detail + - view_full_original_data + - download_full_original_data + - cancel + type: string + x-go-enum-desc: |- + submit UnmaskingActionSubmit 提交申请 + approve UnmaskingActionApprove 批准申请 + reject UnmaskingActionReject 驳回申请 + view_unmasking_workflow_detail UnmaskingActionViewOriginalDataWorkflowDetail 查看工单详情 + view_full_original_data UnmaskingActionViewOriginalData 查看原文 + download_full_original_data UnmaskingActionDownloadOriginalData 下载原文 + cancel UnmaskingActionCancel 取消申请 + x-go-name: Action + action_time: + description: 操作时间 (RFC3339) + type: string + x-go-name: ActionTime + extra_message: + description: 额外信息 (如拦截原因) + type: string + x-go-name: ExtraMessage + operator_name: + description: 操作人姓名 + type: string + x-go-name: OperatorName + operator_uid: + description: 操作人 UID + type: string + x-go-name: OperatorUID + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + UnmaskingSQLDetail: + properties: + lineage_analysis_snapshot: + $ref: '#/definitions/AnalyzeResult' + masked_data: + $ref: '#/definitions/SQLQueryResult' + masking_config_snapshot: + description: 脱敏配置快照 + items: + $ref: '#/definitions/ColumnMaskingConfig' + type: array + x-go-name: MaskingConfigSnapshot + original_data: + $ref: '#/definitions/SQLQueryResult' + sql_content: + description: 原始 SQL 内容 + type: string + x-go-name: SQLContent + sql_index_id: + description: 来源侧 SQL 索引 id + type: string + x-go-name: SQLIndexID + uid: + description: SQL 详情 UID + type: string + x-go-name: UID + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + UnmaskingWorkflowDetail: + properties: + applicant_name: + description: 申请人用户名 + type: string + x-go-name: ApplicantName + apply_reason: + description: 申请理由 + type: string + x-go-name: ApplyReason + approval_status: + description: |- + 审批状态 + pending UnmaskingWorkflowApprovalStatusPending 待审批 + approved UnmaskingWorkflowApprovalStatusApproved 已批准 + rejected UnmaskingWorkflowApprovalStatusRejected 已驳回 + cancelled UnmaskingWorkflowApprovalStatusCancelled 已取消 + enum: + - pending + - approved + - rejected + - cancelled + type: string + x-go-enum-desc: |- + pending UnmaskingWorkflowApprovalStatusPending 待审批 + approved UnmaskingWorkflowApprovalStatusApproved 已批准 + rejected UnmaskingWorkflowApprovalStatusRejected 已驳回 + cancelled UnmaskingWorkflowApprovalStatusCancelled 已取消 + x-go-name: ApprovalStatus + created_at: + description: 申请时间 (RFC3339) + type: string + x-go-name: CreatedAt + current_assignees: + description: 当前待处理人 + items: + $ref: '#/definitions/UidWithName' + type: array + x-go-name: CurrentAssignees + datasource_name: + description: 数据源实例名称 + type: string + x-go-name: DatasourceName + datasource_uid: + description: 数据源实例ID + type: string + x-go-name: DatasourceUid + expire_time: + description: 过期时间 (RFC3339) + type: string + x-go-name: ExpireTime + operation_logs: + description: 操作日志 + items: + $ref: '#/definitions/UnmaskingOperationLogItem' + type: array + x-go-name: OperationLogs + reject_reason: + description: 驳回理由 (整单驳回时) + type: string + x-go-name: RejectReason + source_type: + description: |- + 来源类型 + data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单 + sql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台 + enum: + - data_export + - sql_workbench + type: string + x-go-enum-desc: |- + data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单 + sql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台 + x-go-name: SourceType + source_uid: + description: 来源对象UID + type: string + x-go-name: SourceUID + unmasking_sqls: + description: SQL 详情列表 + items: + $ref: '#/definitions/UnmaskingSQLDetail' + type: array + x-go-name: UnmaskingSQLs + usage_status: + description: |- + 使用情况 + unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看 + viewed UnmaskingWorkflowUsageStatusViewed 已查看 + enum: + - unviewed + - viewed + type: string + x-go-enum-desc: |- + unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看 + viewed UnmaskingWorkflowUsageStatusViewed 已查看 + x-go-name: UsageStatus + workflow_id: + description: 申请编号 + type: string + x-go-name: WorkflowID + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 + UnmaskingWorkflowListItem: + properties: + applicant_name: + description: 申请人用户名 + type: string + x-go-name: ApplicantName + apply_reason: + description: 申请理由 + type: string + x-go-name: ApplyReason + approval_status: + description: |- + 审批状态 + pending UnmaskingWorkflowApprovalStatusPending 待审批 + approved UnmaskingWorkflowApprovalStatusApproved 已批准 + rejected UnmaskingWorkflowApprovalStatusRejected 已驳回 + cancelled UnmaskingWorkflowApprovalStatusCancelled 已取消 + enum: + - pending + - approved + - rejected + - cancelled + type: string + x-go-enum-desc: |- + pending UnmaskingWorkflowApprovalStatusPending 待审批 + approved UnmaskingWorkflowApprovalStatusApproved 已批准 + rejected UnmaskingWorkflowApprovalStatusRejected 已驳回 + cancelled UnmaskingWorkflowApprovalStatusCancelled 已取消 + x-go-name: ApprovalStatus + created_at: + description: 申请时间 (RFC3339) + type: string + x-go-name: CreatedAt + current_assignees: + description: 当前待处理人 + items: + $ref: '#/definitions/UidWithName' + type: array + x-go-name: CurrentAssignees + datasource_name: + description: 数据源实例名称 + type: string + x-go-name: DatasourceName + datasource_uid: + description: 数据源实例ID + type: string + x-go-name: DatasourceUid + expire_time: + description: 过期时间 (RFC3339) + type: string + x-go-name: ExpireTime + source_type: + description: |- + 来源类型 + data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单 + sql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台 + enum: + - data_export + - sql_workbench + type: string + x-go-enum-desc: |- + data_export UnmaskingWorkflowSourceTypeDataExport 数据导出工单 + sql_workbench UnmaskingWorkflowSourceTypeSQLWorkbench SQL工作台 + x-go-name: SourceType + source_uid: + description: 来源对象UID + type: string + x-go-name: SourceUID + usage_status: + description: |- + 使用情况 + unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看 + viewed UnmaskingWorkflowUsageStatusViewed 已查看 + enum: + - unviewed + - viewed + type: string + x-go-enum-desc: |- + unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看 + viewed UnmaskingWorkflowUsageStatusViewed 已查看 + x-go-name: UsageStatus + workflow_id: + description: 申请编号 + type: string + x-go-name: WorkflowID + type: object + x-go-package: github.com/actiontech/dms/api/dms/service/v1 UpdateBusinessTagReq: properties: business_tag: @@ -11336,106 +12070,6 @@ paths: summary: Update an existing environment tag. tags: - Project - /v1/dms/projects/{project_uid}/masking/approval-requests/{request_id}: - get: - operationId: GetPlaintextAccessRequestDetail - parameters: - - description: project uid - example: '"project_uid"' - in: path - name: project_uid - required: true - type: string - x-go-name: ProjectUid - - description: approval request id - example: 1 - format: int64 - in: path - name: request_id - required: true - type: integer - x-go-name: RequestID - responses: - "200": - description: GetPlaintextAccessRequestDetailReply - schema: - $ref: '#/definitions/GetPlaintextAccessRequestDetailReply' - default: - description: GenericResp - schema: - $ref: '#/definitions/GenericResp' - summary: 获取明文访问申请详情。 - tags: - - Masking - /v1/dms/projects/{project_uid}/masking/approval-requests/{request_id}/decisions: - post: - operationId: ProcessApprovalRequest - parameters: - - description: 项目 UID - in: path - name: project_uid - required: true - type: string - - description: 审批申请 ID - in: path - name: request_id - required: true - type: integer - - description: 处理动作信息 - in: body - name: action - required: true - schema: - $ref: '#/definitions/ProcessApprovalRequestReq' - responses: - "200": - description: 成功处理审批申请 - schema: - $ref: '#/definitions/ProcessApprovalRequestReply' - default: - description: 通用错误响应 - schema: - $ref: '#/definitions/GenericResp' - summary: 处理审批申请。 - tags: - - Masking - /v1/dms/projects/{project_uid}/masking/approval-requests/pending: - get: - operationId: ListPendingApprovalRequests - parameters: - - description: project uid - example: '"project_uid"' - in: path - name: project_uid - required: true - type: string - x-go-name: ProjectUid - - description: the maximum count of requests to be returned, default is 20 - example: "20" - format: uint32 - in: query - name: page_size - type: integer - x-go-name: PageSize - - description: the offset of requests to be returned, default is 0 - example: "0" - format: uint32 - in: query - name: page_index - type: integer - x-go-name: PageIndex - responses: - "200": - description: ListPendingApprovalRequestsReply - schema: - $ref: '#/definitions/ListPendingApprovalRequestsReply' - default: - description: GenericResp - schema: - $ref: '#/definitions/GenericResp' - summary: 查询待审批申请列表。 - tags: - - Masking /v1/dms/projects/{project_uid}/masking/overview: get: operationId: GetMaskingOverviewTree @@ -12290,6 +12924,217 @@ paths: summary: 更新脱敏模板。 tags: - Masking + /v1/dms/projects/{project_uid}/masking/unmasking-workflows: + get: + operationId: ListUnmaskingWorkflows + parameters: + - description: project id + in: path + name: project_uid + required: true + type: string + x-go-name: ProjectUid + - description: the maximum count of workflows to be returned + format: uint32 + in: query + name: page_size + required: true + type: integer + x-go-name: PageSize + - description: the offset of workflows to be returned, default is 0 + format: uint32 + in: query + name: page_index + type: integer + x-go-name: PageIndex + - description: filter the approval status + enum: + - pending + - approved + - rejected + - cancelled + in: query + name: filter_by_approval_status + type: string + x-go-enum-desc: |- + pending UnmaskingWorkflowApprovalStatusPending 待审批 + approved UnmaskingWorkflowApprovalStatusApproved 已批准 + rejected UnmaskingWorkflowApprovalStatusRejected 已驳回 + cancelled UnmaskingWorkflowApprovalStatusCancelled 已取消 + x-go-name: FilterByApprovalStatus + - description: filter the usage status + enum: + - unviewed + - viewed + in: query + name: filter_by_usage_status + type: string + x-go-enum-desc: |- + unviewed UnmaskingWorkflowUsageStatusUnviewed 未查看 + viewed UnmaskingWorkflowUsageStatusViewed 已查看 + x-go-name: FilterByUsageStatus + - description: filter db_service id + in: query + name: filter_by_db_service_uid + type: string + x-go-name: FilterByDBServiceUid + responses: + "200": + description: List unmasking workflows successfully + schema: + $ref: '#/definitions/ListUnmaskingWorkflowsReply' + default: + description: Generic error response + schema: + $ref: '#/definitions/GenericResp' + summary: List unmasking workflows. + tags: + - Masking + post: + operationId: CreateUnmaskingWorkflow + parameters: + - description: project id + in: path + name: project_uid + required: true + type: string + - description: unmasking workflow info + in: body + name: unmasking_workflow + required: true + schema: + $ref: '#/definitions/CreateUnmaskingWorkflowReq' + responses: + "200": + description: Create unmasking workflow successfully + schema: + $ref: '#/definitions/CreateUnmaskingWorkflowReply' + default: + description: Generic error response + schema: + $ref: '#/definitions/GenericResp' + summary: Create unmasking workflow. + tags: + - Masking + /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}: + get: + operationId: GetUnmaskingWorkflow + parameters: + - description: project id + in: path + name: project_uid + required: true + type: string + x-go-name: ProjectUid + - description: workflow id + in: path + name: workflow_id + required: true + type: string + x-go-name: WorkflowID + responses: + "200": + description: Get unmasking workflow detail successfully + schema: + $ref: '#/definitions/GetUnmaskingWorkflowReply' + default: + description: Generic error response + schema: + $ref: '#/definitions/GenericResp' + summary: Get unmasking workflow detail. + tags: + - Masking + /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/approve: + post: + operationId: ApproveUnmaskingWorkflow + parameters: + - description: project id + in: path + name: project_uid + required: true + type: string + - description: workflow id + in: path + name: workflow_id + required: true + type: string + - description: approve unmasking workflow info (optional, only carries approve_reason) + in: body + name: approve_unmasking_workflow + schema: + $ref: '#/definitions/ApproveUnmaskingWorkflow' + responses: + "200": + description: Approve unmasking workflow successfully + schema: + $ref: '#/definitions/ApproveUnmaskingWorkflowReply' + default: + description: Generic error response + schema: + $ref: '#/definitions/GenericResp' + summary: Approve unmasking workflow. + tags: + - Masking + /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/cancel: + post: + operationId: CancelUnmaskingWorkflow + parameters: + - description: project id + in: path + name: project_uid + required: true + type: string + x-go-name: ProjectUid + - description: workflow id + in: path + name: workflow_id + required: true + type: string + x-go-name: WorkflowID + responses: + "200": + description: Cancel unmasking workflow successfully + schema: + $ref: '#/definitions/CancelUnmaskingWorkflowReply' + default: + description: Generic error response + schema: + $ref: '#/definitions/GenericResp' + summary: Cancel unmasking workflow. + tags: + - Masking + /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/reject: + post: + operationId: RejectUnmaskingWorkflow + parameters: + - description: project id + in: path + name: project_uid + required: true + type: string + - description: workflow id + in: path + name: workflow_id + required: true + type: string + - description: reject unmasking workflow info + in: body + name: reject_unmasking_workflow + required: true + schema: + $ref: '#/definitions/RejectUnmaskingWorkflow' + responses: + "200": + description: Reject unmasking workflow successfully + schema: + $ref: '#/definitions/RejectUnmaskingWorkflowReply' + default: + description: Generic error response + schema: + $ref: '#/definitions/GenericResp' + summary: Reject unmasking workflow. + tags: + - Masking /v1/dms/projects/{project_uid}/member_groups: get: operationId: ListMemberGroups From e3321d8d9fa7d6711a59401ed5ddadfce7fe9044 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Thu, 14 May 2026 13:39:30 +0000 Subject: [PATCH 22/27] fix(api): add swagger annotations for ListTableColumns API types --- api/dms/service/v1/db_structure_columns.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/dms/service/v1/db_structure_columns.go b/api/dms/service/v1/db_structure_columns.go index 29ee0fff..632e694d 100644 --- a/api/dms/service/v1/db_structure_columns.go +++ b/api/dms/service/v1/db_structure_columns.go @@ -4,6 +4,7 @@ import ( base "github.com/actiontech/dms/pkg/dms-common/api/base/v1" ) +// swagger:parameters ListTableColumns type ListTableColumnsReq struct { // Required: true // in:path @@ -19,12 +20,14 @@ type ListTableColumnsReq struct { TableName string `param:"table_name" json:"table_name" validate:"required"` } +// swagger:model ListTableColumnsReply type ListTableColumnsReply struct { Data []*TableColumn `json:"data"` // Generic reply base.GenericResp } +// swagger:model TableColumn type TableColumn struct { Name string `json:"name"` Type string `json:"type"` From 3eca12e2636bd4b6a5a6dd50edfa3927a389dafd Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Thu, 14 May 2026 14:05:59 +0000 Subject: [PATCH 23/27] fix(data_masking): align unmasking permission verifier signature --- internal/data_masking/biz/unmasking_workflow.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/data_masking/biz/unmasking_workflow.go b/internal/data_masking/biz/unmasking_workflow.go index 27b5f0ad..496e326c 100644 --- a/internal/data_masking/biz/unmasking_workflow.go +++ b/internal/data_masking/biz/unmasking_workflow.go @@ -251,11 +251,11 @@ type UnmaskingOpPermissionRange struct { // UnmaskingWorkflowOpPermissionVerifier 操作权限校验能力。 type UnmaskingWorkflowOpPermissionVerifier interface { IsUserDMSAdmin(ctx context.Context, userUID string) (bool, error) - CanOpGlobal(ctx context.Context, userUID string) (bool, error) + CanOpGlobal(ctx context.Context, userUID string, isBusinessWrite bool) (bool, error) CanViewGlobal(ctx context.Context, userUID string) (bool, error) - IsUserProjectAdmin(ctx context.Context, userUID, projectUID string) (bool, error) + IsUserProjectAdmin(ctx context.Context, userUID, projectUID string, isBusinessWrite bool) (bool, error) GetUserOpPermissionInProject(ctx context.Context, userUID, projectUID string) ([]UnmaskingOpPermissionRange, error) - GetCanOpDBUsers(ctx context.Context, projectUID, dbServiceUID string, needOpPermissionTypes []string) ([]string, error) + GetCanOpDBUsers(ctx context.Context, projectUID, dbServiceUID string, needOpPermissionTypes []string, isBusinessWrite bool) ([]string, error) } // UnmaskingWorkflowUserDirectory 用户领域：工单列表/详情等场景将用户 UID 解析为展示名。 From b310763280f282a1e5764f9884da8e24ef73f5d3 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Fri, 15 May 2026 07:56:53 +0000 Subject: [PATCH 24/27] feat(data_export): add API for downloading unmasked SQL query results and enhance masking rules listing --- api/dms/service/v1/data_export_workflow.go | 7 +++ .../apiserver/service/data_mask_controller.go | 50 +++++++++++++++++++ internal/apiserver/service/dms_controller.go | 26 ++++++++++ internal/apiserver/service/router.go | 1 + 4 files changed, 84 insertions(+) diff --git a/api/dms/service/v1/data_export_workflow.go b/api/dms/service/v1/data_export_workflow.go index c2721ab5..6ff33cb5 100644 --- a/api/dms/service/v1/data_export_workflow.go +++ b/api/dms/service/v1/data_export_workflow.go @@ -282,6 +282,13 @@ type DownloadOriginalDataExportWorkflowReq struct { UnmaskingWorkflowUid string `query:"unmasking_workflow_uid" json:"unmasking_workflow_uid" validate:"required"` } +// swagger:response DownloadOriginalDataExportWorkflowReply +type DownloadOriginalDataExportWorkflowReply struct { + // swagger:file + // in: body + File []byte +} + type RejectDataExportWorkflowPayload struct { // Required: true Reason string `json:"reason" validate:"required"` diff --git a/internal/apiserver/service/data_mask_controller.go b/internal/apiserver/service/data_mask_controller.go index fc032186..2a86e847 100644 --- a/internal/apiserver/service/data_mask_controller.go +++ b/internal/apiserver/service/data_mask_controller.go @@ -31,6 +31,50 @@ func (ctl *DMSController) ListMaskingRules(c echo.Context) error { return NewOkRespWithReply(c, reply) } +// swagger:operation GET /v1/dms/masking/rules Masking ListMaskingRulesWithoutProject +// +// 查询脱敏规则列表（兼容旧入口，仅返回内置规则，可携带筛选参数）。 +// +// --- +// parameters: +// - name: source +// description: 规则来源筛选，builtin 或 custom，为空时返回全部 +// in: query +// type: string +// - name: keywords +// description: 模糊搜索关键字，匹配规则名称、描述或敏感数据类型名称 +// in: query +// type: string +// - name: page_size +// description: 分页大小，默认 20 +// in: query +// type: integer +// - name: page_index +// description: 页码（从1开始），默认 1 +// in: query +// type: integer +// responses: +// '200': +// description: 查询脱敏规则列表成功 +// schema: +// "$ref": "#/definitions/ListMaskingRulesReply" +// default: +// description: 通用错误响应 +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) ListMaskingRulesWithoutProject(c echo.Context) error { + req := &aV1.ListMaskingRulesReq{} + if err := c.Bind(req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + + reply, err := ctl.DMS.ListMaskingRules(c.Request().Context(), req) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + return NewOkRespWithReply(c, reply) +} + // swagger:route GET /v1/dms/projects/{project_uid}/masking/templates Masking ListMaskingTemplates // // 查询脱敏模板列表。 @@ -51,7 +95,13 @@ func (ctl *DMSController) ListMaskingTemplates(c echo.Context) error { return NewOkRespWithReply(c, reply) } +// swagger:route GET /v1/dms/projects/{project_uid}/db_services/{db_service_uid}/schemas/{schema_name}/tables/{table_name}/columns DBStructure ListTableColumns +// // List table columns (internal API for lineage analysis). +// +// responses: +// 200: body:ListTableColumnsReply +// default: body:GenericResp func (ctl *DMSController) ListTableColumns(c echo.Context) error { // 内部接口，仅允许sys/admin用户访问 currentUserUid, err := jwt.GetUserUidStrFromContext(c) diff --git a/internal/apiserver/service/dms_controller.go b/internal/apiserver/service/dms_controller.go index 5582b58e..d0d77577 100644 --- a/internal/apiserver/service/dms_controller.go +++ b/internal/apiserver/service/dms_controller.go @@ -3976,6 +3976,32 @@ func (ctl *DMSController) ExportDataExportWorkflow(c echo.Context) error { return NewOkResp(c) } +// swagger:route GET /v1/dms/projects/{project_uid}/data_export_workflows/{data_export_workflow_uid}/original-export/download DataExportWorkflows DownloadOriginalDataExportWorkflow +// +// Download unmasked SQL query results as a zip file. Each request runs export in memory; files are not persisted. +// +// responses: +// 200: DownloadOriginalDataExportWorkflowReply +// default: body:GenericResp +func (ctl *DMSController) DownloadOriginalDataExportWorkflow(c echo.Context) error { + req := new(aV1.DownloadOriginalDataExportWorkflowReq) + if err := bindAndValidateReq(c, req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + fileName, content, err := ctl.DMS.DownloadOriginalDataExportWorkflow(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + c.Response().Header().Set(echo.HeaderContentDisposition, + mime.FormatMediaType("attachment", map[string]string{"filename": fileName})) + + return c.Blob(http.StatusOK, "application/zip", content) +} + // swagger:operation POST /v1/dms/projects/{project_uid}/data_export_tasks DataExportTask AddDataExportTask // // Add data_export task. diff --git a/internal/apiserver/service/router.go b/internal/apiserver/service/router.go index 6acdf54d..a2e5cdba 100644 --- a/internal/apiserver/service/router.go +++ b/internal/apiserver/service/router.go @@ -225,6 +225,7 @@ func (s *APIServer) initRouter() error { dataExportWorkflowsV1.POST("/:data_export_workflow_uid/approve", s.DMSController.ApproveDataExportWorkflow) dataExportWorkflowsV1.POST("/:data_export_workflow_uid/reject", s.DMSController.RejectDataExportWorkflow) dataExportWorkflowsV1.POST("/:data_export_workflow_uid/export", s.DMSController.ExportDataExportWorkflow) + dataExportWorkflowsV1.GET("/:data_export_workflow_uid/original-export/download", s.DMSController.DownloadOriginalDataExportWorkflow) dataExportWorkflowsV1.POST("/cancel", s.DMSController.CancelDataExportWorkflow) // 内部接口，仅允许sys用户访问 From 76c61f6ab560764ae26606e24d3868dc2b2dc079 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Mon, 18 May 2026 06:01:43 +0000 Subject: [PATCH 25/27] feat(data_export): add unmasking workflow details to data export workflow model --- api/dms/service/v1/data_export_workflow.go | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/api/dms/service/v1/data_export_workflow.go b/api/dms/service/v1/data_export_workflow.go index 6ff33cb5..f4d2d6d1 100644 --- a/api/dms/service/v1/data_export_workflow.go +++ b/api/dms/service/v1/data_export_workflow.go @@ -3,6 +3,7 @@ package v1 import ( "time" + "github.com/actiontech/dms/internal/data_masking/biz" base "github.com/actiontech/dms/pkg/dms-common/api/base/v1" dmsCommonV1 "github.com/actiontech/dms/pkg/dms-common/api/dms/v1" ) @@ -221,6 +222,31 @@ type GetDataExportWorkflow struct { CreateTime *time.Time `json:"create_time"` WorkflowRecord WorkflowRecord `json:"workflow_record"` WorkflowRecordHistory []WorkflowRecord `json:"workflow_record_history"` + // UnmaskingWorkflow 关联的查看原文工单摘要；无关联时为 null + UnmaskingWorkflow *DataExportRelatedUnmaskingWorkflow `json:"unmasking_workflow"` +} + +// swagger:model DataExportRelatedUnmaskingWorkflow +// DataExportRelatedUnmaskingWorkflow 数据导出工单关联的查看原文工单（仅摘要字段） +type DataExportRelatedUnmaskingWorkflow struct { + // UnmaskingWorkflowUid 查看原文工单 UID，用于跳转详情等 + UnmaskingWorkflowUid string `json:"unmasking_workflow_uid"` + // 创建人（uid + 展示名） + Creator UidWithName `json:"creator"` + // 创建时间 (RFC3339) + CreatedAt string `json:"created_at"` + // 过期时间 (RFC3339)，未设置时省略 + ExpireTime string `json:"expire_time,omitempty"` + // 审批状态 + ApprovalStatus biz.UnmaskingWorkflowApprovalStatus `json:"approval_status"` + // 使用状态 + UsageStatus biz.UnmaskingWorkflowUsageStatus `json:"usage_status"` + // 申请理由 + ApplyReason string `json:"apply_reason"` + // 驳回理由（整单驳回时） + RejectReason string `json:"reject_reason,omitempty"` + // 操作记录 + OperationLogs []*UnmaskingOperationLogItem `json:"operation_logs"` } type WorkflowRecord struct { From 14cb2ce8795949bf34b7c52c6be2b1a77dbf2599 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Mon, 18 May 2026 07:10:34 +0000 Subject: [PATCH 26/27] feat(data_export): add function to check if data export workflow status allows original download --- internal/dms/biz/data_export_workflow.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/dms/biz/data_export_workflow.go b/internal/dms/biz/data_export_workflow.go index be58e706..96b19eeb 100644 --- a/internal/dms/biz/data_export_workflow.go +++ b/internal/dms/biz/data_export_workflow.go @@ -30,6 +30,10 @@ func (dews DataExportWorkflowStatus) String() string { return string(dews) } +func isDataExportWorkflowStatusAllowDownloadOriginal(status DataExportWorkflowStatus) bool { + return status == DataExportWorkflowStatusFinish +} + type EventType string const ( From 7f933145947a4e9ff999e4e683a6131b19191347 Mon Sep 17 00:00:00 2001 From: yangzhongjiao Date: Tue, 19 May 2026 01:55:25 +0000 Subject: [PATCH 27/27] feat(unmasking): implement activation and plaintext retrieval for unmasking workflows with new request and response models --- api/dms/service/v1/unmasking_workflow.go | 64 +++++++++++++++- .../apiserver/service/data_mask_controller.go | 76 +++++++++++++++++++ .../data_masking/biz/unmasking_workflow.go | 17 +++++ internal/dms/service/unmasking_workflow_ce.go | 8 ++ 4 files changed, 164 insertions(+), 1 deletion(-) diff --git a/api/dms/service/v1/unmasking_workflow.go b/api/dms/service/v1/unmasking_workflow.go index 633f915b..6cc06774 100644 --- a/api/dms/service/v1/unmasking_workflow.go +++ b/api/dms/service/v1/unmasking_workflow.go @@ -57,8 +57,18 @@ type UnmaskingWorkflowListItem struct { ApprovalStatus biz.UnmaskingWorkflowApprovalStatus `json:"approval_status" validate:"oneof=pending approved rejected cancelled"` // 使用情况 UsageStatus biz.UnmaskingWorkflowUsageStatus `json:"usage_status" validate:"oneof=unviewed viewed"` - // 过期时间 (RFC3339) + // 过期时间 (RFC3339)，兼容字段：批准前为激活截止，激活后为查看截止 ExpireTime string `json:"expire_time" example:"2024-01-16T10:30:00Z"` + // 激活查看时刻 (RFC3339) + ActivatedAt string `json:"activated_at"` + // 须在此时刻前激活查看 (RFC3339) + ActivationDeadline string `json:"activation_deadline"` + // 明文查看截止 (RFC3339) + ViewValidUntil string `json:"view_valid_until"` + // 申请人明文查看状态 + ViewState biz.UnmaskingWorkflowViewState `json:"view_state"` + // 是否可点击激活查看 + CanActivate bool `json:"can_activate"` // 申请理由 ApplyReason string `json:"apply_reason"` // 当前待处理人 @@ -252,3 +262,55 @@ type CancelUnmaskingWorkflowReply struct { // Generic reply base.GenericResp } + +// swagger:parameters ActivateUnmaskingWorkflowView +type ActivateUnmaskingWorkflowViewReq struct { + // swagger:ignore + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // in: path + // Required: true + WorkflowID string `param:"workflow_id" json:"workflow_id" validate:"required"` +} + +// swagger:model ActivateUnmaskingWorkflowViewReply +type ActivateUnmaskingWorkflowViewReply struct { + Data *ActivateUnmaskingWorkflowViewReplyData `json:"data"` + base.GenericResp +} + +// swagger:model ActivateUnmaskingWorkflowViewReplyData +type ActivateUnmaskingWorkflowViewReplyData struct { + ViewValidUntil string `json:"view_valid_until"` + ViewState biz.UnmaskingWorkflowViewState `json:"view_state"` +} + +// swagger:parameters GetUnmaskingWorkflowPlaintext +type GetUnmaskingWorkflowPlaintextReq struct { + // swagger:ignore + ProjectUid string `param:"project_uid" json:"project_uid" validate:"required"` + // in: path + // Required: true + WorkflowID string `param:"workflow_id" json:"workflow_id" validate:"required"` +} + +// swagger:model GetUnmaskingWorkflowPlaintextReply +type GetUnmaskingWorkflowPlaintextReply struct { + Data *GetUnmaskingWorkflowPlaintext `json:"data"` + base.GenericResp +} + +// swagger:model GetUnmaskingWorkflowPlaintext +type GetUnmaskingWorkflowPlaintext struct { + ViewState biz.UnmaskingWorkflowViewState `json:"view_state"` + ViewValidUntil string `json:"view_valid_until"` + UnmaskingSQLs []*UnmaskingPlaintextSQLItem `json:"unmasking_sqls"` +} + +// swagger:model UnmaskingPlaintextSQLItem +type UnmaskingPlaintextSQLItem struct { + UID string `json:"uid"` + SQLIndexID string `json:"sql_index_id"` + OriginalData *SQLQueryResult `json:"original_data"` + MaskedColumns []string `json:"masked_columns"` + Truncated bool `json:"truncated"` +} diff --git a/internal/apiserver/service/data_mask_controller.go b/internal/apiserver/service/data_mask_controller.go index 2a86e847..9ad84b9f 100644 --- a/internal/apiserver/service/data_mask_controller.go +++ b/internal/apiserver/service/data_mask_controller.go @@ -857,6 +857,82 @@ func (ctl *DMSController) CancelUnmaskingWorkflow(c echo.Context) error { return NewOkRespWithReply(c, &aV1.CancelUnmaskingWorkflowReply{}) } +// swagger:operation POST /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/activate Masking ActivateUnmaskingWorkflowView +// +// Activate plaintext view window for applicant. +// +// --- +// parameters: +// - name: project_uid +// in: path +// required: true +// type: string +// - name: workflow_id +// in: path +// required: true +// type: string +// +// responses: +// '200': +// schema: +// "$ref": "#/definitions/ActivateUnmaskingWorkflowViewReply" +// default: +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) ActivateUnmaskingWorkflowView(c echo.Context) error { + req := &aV1.ActivateUnmaskingWorkflowViewReq{} + if err := bindAndValidateReq(c, req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + reply, err := ctl.DMS.ActivateUnmaskingWorkflowView(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + return NewOkRespWithReply(c, reply) +} + +// swagger:operation GET /v1/dms/projects/{project_uid}/masking/unmasking-workflows/{workflow_id}/plaintext Masking GetUnmaskingWorkflowPlaintext +// +// Get plaintext query snapshot during active view window. +// +// --- +// parameters: +// - name: project_uid +// in: path +// required: true +// type: string +// - name: workflow_id +// in: path +// required: true +// type: string +// +// responses: +// '200': +// schema: +// "$ref": "#/definitions/GetUnmaskingWorkflowPlaintextReply" +// default: +// schema: +// "$ref": "#/definitions/GenericResp" +func (ctl *DMSController) GetUnmaskingWorkflowPlaintext(c echo.Context) error { + req := &aV1.GetUnmaskingWorkflowPlaintextReq{} + if err := bindAndValidateReq(c, req); err != nil { + return NewErrResp(c, err, apiError.BadRequestErr) + } + currentUserUid, err := jwt.GetUserUidStrFromContext(c) + if err != nil { + return NewErrResp(c, err, apiError.UnauthorizedErr) + } + reply, err := ctl.DMS.GetUnmaskingWorkflowPlaintext(c.Request().Context(), req, currentUserUid) + if err != nil { + return NewErrResp(c, err, apiError.DMSServiceErr) + } + return NewOkRespWithReply(c, reply) +} + // swagger:route GET /v1/dms/projects/{project_uid}/masking/rules/{rule_id} Masking GetMaskingRuleDetail // // 获取脱敏规则详情。 diff --git a/internal/data_masking/biz/unmasking_workflow.go b/internal/data_masking/biz/unmasking_workflow.go index 496e326c..3a8f2023 100644 --- a/internal/data_masking/biz/unmasking_workflow.go +++ b/internal/data_masking/biz/unmasking_workflow.go @@ -229,12 +229,29 @@ const ( UnmaskingActionDownloadOriginalData UnmaskingAction = "download_full_original_data" // 取消申请 UnmaskingActionCancel UnmaskingAction = "cancel" + // 激活查看原文（工单详情） + UnmaskingActionActivateView UnmaskingAction = "activate_view" ) func (a UnmaskingAction) String() string { return string(a) } +// UnmaskingWorkflowViewState 申请人明文查看状态（详情页展示） +// swagger:enum UnmaskingWorkflowViewState +type UnmaskingWorkflowViewState string + +const ( + UnmaskingWorkflowViewStateNotActivated UnmaskingWorkflowViewState = "not_activated" + UnmaskingWorkflowViewStateActive UnmaskingWorkflowViewState = "active" + UnmaskingWorkflowViewStateViewExpired UnmaskingWorkflowViewState = "view_expired" + UnmaskingWorkflowViewStateActivationExpired UnmaskingWorkflowViewState = "activation_expired" +) + +func (s UnmaskingWorkflowViewState) String() string { + return string(s) +} + // 与 DMS OpRangeType 字符串取值一致，用于 UnmaskingOpPermissionRange 判定。 const ( UnmaskingOpRangeProject = "project" diff --git a/internal/dms/service/unmasking_workflow_ce.go b/internal/dms/service/unmasking_workflow_ce.go index 4cf7bd29..07923338 100644 --- a/internal/dms/service/unmasking_workflow_ce.go +++ b/internal/dms/service/unmasking_workflow_ce.go @@ -44,3 +44,11 @@ func (d *DMSService) RejectUnmaskingWorkflow(ctx context.Context, req *v1.Reject func (d *DMSService) CancelUnmaskingWorkflow(ctx context.Context, req *v1.CancelUnmaskingWorkflowReq, currentUserUid string) error { return errNotSupportUnmaskingWorkflow } + +func (d *DMSService) ActivateUnmaskingWorkflowView(ctx context.Context, req *v1.ActivateUnmaskingWorkflowViewReq, currentUserUid string) (*v1.ActivateUnmaskingWorkflowViewReply, error) { + return nil, errNotSupportUnmaskingWorkflow +} + +func (d *DMSService) GetUnmaskingWorkflowPlaintext(ctx context.Context, req *v1.GetUnmaskingWorkflowPlaintextReq, currentUserUid string) (*v1.GetUnmaskingWorkflowPlaintextReply, error) { + return nil, errNotSupportUnmaskingWorkflow +}