From 1351e5093380a5a7b3e918267751a9dea68d407f Mon Sep 17 00:00:00 2001 From: Xianquan Date: Mon, 6 Jul 2026 16:28:43 +0800 Subject: [PATCH] feat(s3): add short access links --- .../design/api/s3-object-key-authorization.md | 2 +- .../common/s3/shortlink-access-token.md | 963 ++++++++++++++++++ packages/global/common/string/tools.ts | 2 +- .../global/openapi/core/dataset/data/api.ts | 2 +- .../service/common/s3/accessLink/constants.ts | 21 + .../s3/accessLink/downloadAlias/entity.ts | 102 ++ .../s3/accessLink/downloadAlias/schema.ts | 59 ++ .../s3/accessLink/downloadAlias/service.ts | 102 ++ .../service/common/s3/accessLink/error.ts | 26 + .../service/common/s3/accessLink/index.ts | 7 + packages/service/common/s3/accessLink/type.ts | 106 ++ .../s3/accessLink/uploadSession/entity.ts | 43 + .../s3/accessLink/uploadSession/schema.ts | 54 + .../s3/accessLink/uploadSession/service.ts | 78 ++ .../service/common/s3/accessLink/utils.ts | 169 +++ packages/service/common/s3/buckets/base.ts | 24 +- packages/service/common/s3/queue/delete.ts | 12 + packages/service/common/s3/security/token.ts | 82 -- .../service/core/dataset/data/controller.ts | 43 +- .../search/defaultRecall/embeddingRecall.ts | 55 +- .../search/defaultRecall/fullTextRecall.ts | 69 +- .../dataset/search/defaultRecall/index.ts | 10 +- .../dataset/search/defaultRecall/result.ts | 25 +- packages/service/core/dataset/utils.ts | 10 +- packages/service/core/workflow/utils/file.ts | 2 +- .../service/test/common/s3/accessLink.test.ts | 214 ++++ packages/service/test/common/s3/token.test.ts | 51 +- .../test/core/dataset/data/controller.test.ts | 4 +- .../service/test/core/dataset/utils.test.ts | 327 +++--- .../core/chat/components/FormInputResult.tsx | 2 +- .../core/chat/record/getCollectionQuote.ts | 9 +- .../pages/api/core/chat/record/getQuote.ts | 2 +- .../api/core/dataset/collection/export.ts | 28 +- .../api/core/dataset/data/getQuoteData.ts | 8 +- .../src/pages/api/core/dataset/data/update.ts | 9 +- .../pages/api/core/dataset/data/v2/list.ts | 18 +- .../api/core/dataset/file/getPreviewChunks.ts | 10 +- .../file/getSearchTestImagePreviewUrls.ts | 24 +- .../dataset/training/getTrainingDataDetail.ts | 20 +- .../pages/api/system/file/d/[signedAlias].ts | 31 + .../pages/api/system/file/download/[token].ts | 110 +- .../src/pages/api/system/file/u/[token].ts | 37 + .../pages/api/system/file/upload/[token].ts | 228 +---- projects/app/src/service/common/s3/proxy.ts | 402 ++++++++ .../test/api/system/file/accessLink.test.ts | 400 ++++++++ .../api/system/file/sourceContentType.test.ts | 98 +- 46 files changed, 3376 insertions(+), 724 deletions(-) create mode 100644 .agents/design/common/s3/shortlink-access-token.md create mode 100644 packages/service/common/s3/accessLink/constants.ts create mode 100644 packages/service/common/s3/accessLink/downloadAlias/entity.ts create mode 100644 packages/service/common/s3/accessLink/downloadAlias/schema.ts create mode 100644 packages/service/common/s3/accessLink/downloadAlias/service.ts create mode 100644 packages/service/common/s3/accessLink/error.ts create mode 100644 packages/service/common/s3/accessLink/index.ts create mode 100644 packages/service/common/s3/accessLink/type.ts create mode 100644 packages/service/common/s3/accessLink/uploadSession/entity.ts create mode 100644 packages/service/common/s3/accessLink/uploadSession/schema.ts create mode 100644 packages/service/common/s3/accessLink/uploadSession/service.ts create mode 100644 packages/service/common/s3/accessLink/utils.ts create mode 100644 packages/service/test/common/s3/accessLink.test.ts create mode 100644 projects/app/src/pages/api/system/file/d/[signedAlias].ts create mode 100644 projects/app/src/pages/api/system/file/u/[token].ts create mode 100644 projects/app/src/service/common/s3/proxy.ts create mode 100644 projects/app/test/api/system/file/accessLink.test.ts diff --git a/.agents/design/api/s3-object-key-authorization.md b/.agents/design/api/s3-object-key-authorization.md index ea9d324530bd..9aa820616d2a 100644 --- a/.agents/design/api/s3-object-key-authorization.md +++ b/.agents/design/api/s3-object-key-authorization.md @@ -50,7 +50,7 @@ FastGPT 的私有对象存储 key 是 bucket 内的全局路径字符串，例 2. 先完成业务资源鉴权，拿到可信的 `teamId`、`appId`、`datasetId`、`uid` 或 `userId`。 3. 使用对应 `isAuthorized*FileS3Key` helper 绑定 key 与可信上下文。 4. 绑定失败时返回通用未授权错误，不暴露 key 是否存在。 -5. 只有通过绑定后，才允许调用 `createExternalUrl`、`createGet*URL`、`jwtSignS3DownloadToken`、`downloadObject`、`getDatasetFileRawText`、`isObjectExists` 等存储层能力。 +5. 只有通过绑定后，才允许调用 `createExternalUrl`、`createGet*URL`、`createS3DownloadAccessUrl`、`downloadObject`、`getDatasetFileRawText`、`isObjectExists` 等存储层能力。 ## 底层防线 diff --git a/.agents/design/common/s3/shortlink-access-token.md b/.agents/design/common/s3/shortlink-access-token.md new file mode 100644 index 000000000000..9d2227a80230 --- /dev/null +++ b/.agents/design/common/s3/shortlink-access-token.md @@ -0,0 +1,963 @@ +# S3 短链访问凭证设计 + +## 0. 文档标识 + +- 任务前缀：`s3-refactor-shortlink` +- 文档文件名：`shortlink-access-token.md` +- 更新时间：2026-07-03 +- 推荐 PR：PR 1 +- 优先级：P0 + +## 1. 需求背景 + +当前 proxy 上传/下载 URL 把 JWT 放在 path 中： + +```text +/api/system/file/download/ +/api/system/file/upload/ +``` + +下载 JWT 包含 `objectKey/bucketName/type/exp` 等信息；上传 JWT 还包含 `maxSize/uploadConstraints/metadata`。链接很长，AI 大模型在引用或复述时容易替换、截断或重排 JWT 中的字符，导致文件无法预览或上传。 + +## 2. 核心判断 + +短链本质上是在三件事之间取舍： + +1. URL 足够短，不能把完整 payload 塞回 URL。 +2. 访问凭证必须有过期时间，不能因为短链而变成永久公开链接。 +3. 不能让页面每次打开、列表每次读取、markdown 每次渲染都制造一条新 Mongo 文档。 + +因此 PR1 不建议把 download/preview 做成“每次签发一个随机 token，然后用 token hash 查询 Mongo payload 文档”。这种方案适合验证码、邀请链接、一次性魔法链接，但不适合高频图片预览。 + +推荐改为混合方案： + +| 场景 | 推荐模型 | 原因 | +|---|---|---| +| download/preview | object alias + URL 内短 expiry + HMAC 签名 | 不为每次签发创建 token 文档，文档数接近对象数 | +| upload | 短 upload session token + Mongo TTL 文档 | 上传 payload 大且语义是一次上传会话 | + +## 3. 目标 + +1. 新签发的 proxy 上传/下载链接改为短 URL。 +2. download URL 不包含 objectKey、bucketName、长 JWT payload。 +3. download 不按每次签发写 token 文档，只维护对象 alias 文档。 +4. upload 使用短 session token，完整上传策略仍在服务端保存。 +5. 旧 JWT 路由继续兼容，已签发旧链接在过期前可用。 +6. 不改变业务授权边界。objectKey 必须在签发前由业务入口完成归属校验。 + +目标 URL： + +```text +/api/system/file/d/.. +/api/system/file/u/ +``` + +示例长度： + +```text +/api/system/file/d/R7mQG0Yh2kVxP9Za.hovs2.KuH3qfpyxW8A4FnTnaZrJw +/api/system/file/u/wZ6rm5X4l6Ygm1oDQX8JbA +``` + +## 4. 非目标 + +1. 不调整上传文件类型校验。 +2. 不调整 ChatBox 上传取消逻辑。 +3. 不删除 `packages/service/common/s3/security/token.ts` 中旧 JWT 校验兼容能力。 +4. 不把业务权限校验下沉到通用 S3 proxy。 +5. 不保证历史回答里已经过期的链接永久可访问。短链只解决链接过长和易被模型改写的问题，不改变访问凭证的过期语义。 + +## 5. 当前代码中的重复签发路径 + +当前代码里存在大量“页面打开/列表读取/markdown 渲染时重新签发预览链接”的路径，例如： + +1. `replaceS3KeyToPreviewUrl` 会把 dataset/chat/temp objectKey 替换成下载 URL。 +2. `presignVariablesFileUrls` 会在 chat init/outLink init 时给变量文件重新签发预览 URL。 +3. `presignChatFileGetUrl` 会按需给 Chat 文件签发下载 URL。 +4. `createPresignedPutUrl` 上传成功前也会返回一个 `previewUrl`。 + +如果 download 短链实现成“每次签发生成随机 token，并插入一条 Mongo 记录”，Mongo 文档数会随着访问次数膨胀。这个风险不能靠后续清理完全解决，因为高峰期写入压力会先出现。 + +## 6. download/preview 设计 + +### 6.1 URL 格式 + +```text +/api/system/file/d/.. +``` + +字段含义： + +| 字段 | 说明 | +|---|---| +| `aliasId` | URL-safe 随机 id，映射到 `bucketName/objectKey` | +| `expMinute36` | 过期时间，使用 unix minute 的 base36 表示 | +| `sig` | HMAC-SHA256 签名截断后 base64url 编码 | + +签名内容： + +```ts +const signingInput = `s3-download:v1:${aliasId}:${expMinute36}`; +const sig = hmacSha256Base64Url(serviceEnv.FILE_TOKEN_KEY, signingInput).slice(0, 22); +``` + +设计原因： + +1. URL 中只有 alias、短过期时间和签名，不出现 objectKey。 +2. 过期时间在 URL 中，过期请求可以先拒绝，不必查 Mongo。 +3. HMAC 能防止用户篡改 `expMinute36`。 +4. Mongo 不存每次签发的 token，因此不存在“签发次数等于文档数”的问题。 + +### 6.2 alias 文档 + +`packages/service/common/s3/accessLink/downloadAlias/schema.ts` + +```ts +export type S3DownloadAliasType = { + _id: string; + aliasId: string; + aliasKey: string; + bucketName: string; + objectKey: string; + filename?: string; + responseContentType?: string; + createTime: Date; + updateTime: Date; + lastIssuedAt: Date; + purgeAt: Date; + disabledAt?: Date; +}; +``` + +字段说明： + +| 字段 | 用途 | +|---|---| +| `aliasId` | 短 URL 中的资源 id，唯一索引 | +| `aliasKey` | `bucketName/objectKey/filename/responseContentType` 的 HMAC，唯一索引，用于签发时去重 | +| `bucketName/objectKey` | S3 实际读取目标 | +| `filename/responseContentType` | 下载响应的稳定变体参数 | +| `lastIssuedAt` | 最近一次签发时间，只做观测和低频更新 | +| `purgeAt` | alias 文档清理时间，TTL 索引 | +| `disabledAt` | 撤销 alias，撤销后所有该对象 alias 的下载链接失效 | + +建议索引： + +```ts +S3DownloadAliasSchema.index({ aliasId: 1 }, { unique: true }); +S3DownloadAliasSchema.index({ aliasKey: 1 }, { unique: true }); +S3DownloadAliasSchema.index({ purgeAt: 1 }, { expireAfterSeconds: 0 }); +``` + +### 6.3 签发流程 + +```text +createS3DownloadAccessUrl + -> 计算 aliasKey + -> findOne(aliasKey) + -> 不存在则创建 aliasId + -> 计算 expMinute36 + -> 用 $max 更新 purgeAt，确保 alias 至少保留到签发链接过期之后 + -> 返回 /api/system/file/d/.. +``` + +重点： + +1. 同一个 `bucketName/objectKey/filename/responseContentType` 只会有一个 alias 文档。 +2. 页面重复打开只会复用 alias，不会新建 token 文档。 +3. `expMinute36` 可以按时间桶取整，减少 URL 抖动和 `purgeAt` 更新频率。 +4. 并发创建同一个 alias 时，用 `aliasKey` 唯一索引兜底；遇到 duplicate key 后重新查询即可。 + +### 6.4 过期时间分桶 + +URL 里有 `expMinute36`，所以即使每次签发的过期时间不同，也不会产生新文档。但分桶仍然有价值： + +1. 减少同一页面反复渲染时 URL 变化。 +2. 减少 `$max(purgeAt)` 更新频率。 +3. 提升浏览器、CDN、前端 memo 的命中概率。 + +建议规则： + +| requested TTL | exp 取整粒度 | 说明 | +|---|---|---| +| <= 2 小时 | 5 到 15 分钟 | 普通预览链接 | +| <= 24 小时 | 30 到 60 分钟 | chat/fileSelect 预览 | +| > 24 小时 | 1 天 | dataset 引用、导出、长上下文引用 | + +取整方向使用向上取整，保证返回链接不会短于调用方请求的 TTL。 + +### 6.5 验证流程 + +```text +handle /api/system/file/d/[signedAlias] + -> parse aliasId/expMinute36/sig + -> 检查 expMinute36 是否过期，过期直接拒绝 + -> 重新计算 HMAC，constant-time 对比 sig + -> 按 aliasId 查询 Mongo alias + -> 检查 disabledAt + -> 读取 bucketName/objectKey 并代理输出 stream +``` + +这个流程把无效或过期请求挡在 Mongo 查询之前，只让签名正确且未过期的请求进入 alias 查询。 + +### 6.6 清理策略 + +这里不要试图判断“短链是不是不再使用了”。从第一性原理看，系统无法知道一个 bearer URL 未来还会不会被用户、浏览器缓存、模型回答、第三方页面再次访问。能可靠判断的只有： + +1. 这个 URL 是否已经过期。 +2. 这个 alias 是否被主动撤销。 +3. 这个 alias 是否已经超过保留时间。 + +因此清理策略是 lease-based，而不是 usage-based： + +| 对象 | 清理条件 | 说明 | +|---|---|---| +| download URL | `expMinute36 < now` 后自然失效 | URL 失效不需要删 DB，因为没有 per-URL 文档 | +| download alias 文档 | `purgeAt <= now` 由 Mongo TTL 删除 | `purgeAt` 至少晚于最后签发链接的过期时间 | +| revoked alias | `disabledAt` 后立即拒绝访问 | 可保留到 `purgeAt` 由 TTL 清理 | +| S3 object | 沿用现有 `s3_ttls` 和删除队列 | alias 不负责对象生命周期 | + +`purgeAt` 建议计算： + +```ts +const purgeAt = addHours(expiredAtFromUrl, 24); +``` + +签发时使用 `$max: { purgeAt }`，这样： + +1. alias 不会在已签发 URL 过期前被 TTL 删除。 +2. 如果对象长期没人再签发，最后一个链接过期后 alias 会自动清理。 +3. 如果对象长期频繁被查看，只有同一个 alias 文档的 `purgeAt` 被推进，不会产生新文档。 + +不要用 `lastUsedAt` 来决定删除，也不要访问一次就续期 URL。否则泄漏链接可能因为正常页面访问而间接延长有效期，安全语义比当前 JWT 更弱。 + +### 6.7 和对象删除的关系 + +alias 文档不应该成为 S3 对象生命周期的主状态源： + +1. 如果对象被删除但 alias 还没 TTL 清理，请求会在读取 S3 时失败。 +2. `S3BaseBucket.removeObject` 和删除队列可以 best-effort 删除对应 alias，减少孤儿 alias。 +3. 即使 best-effort 删除遗漏，`purgeAt` 仍会兜底清理 alias 文档。 + +PR1 不要求一次性覆盖所有历史删除路径，但需要把 `deleteDownloadAliasByObject({ bucketName, objectKey })` 作为公共函数放好，后续删除链路可以逐步接入。 + +### 6.8 download alias 文档生命周期 + +| 动作 | 触发时机 | DB 行为 | +|---|---|---| +| 插入 | 第一次为某个 `bucketName/objectKey/filename/responseContentType` 签发 download URL | 创建 alias 文档，写入 `aliasId/aliasKey/bucketName/objectKey/purgeAt` | +| 复用 | 后续再次为同一个资源变体签发 download URL | 不插入新文档，只复用已有 `aliasId` | +| 延长保留 | 新签发 URL 的过期时间晚于当前 `purgeAt` 保护范围 | 用 `$max` 推进 `purgeAt`，避免已签发 URL 还没过期时 alias 被 TTL 删除 | +| 主动撤销 | 管理操作或安全事件需要让 alias 立即失效 | 写入 `disabledAt`，访问立即拒绝，文档仍可等 `purgeAt` TTL 清理 | +| 主动删除 | S3 对象被删除、迁移或确认不再可访问 | best-effort 删除对应 alias 文档 | +| 自动清理 | `purgeAt <= now` | Mongo TTL index 自动删除文档 | + +注意：Mongo TTL 删除不是实时语义，它由 MongoDB TTL monitor 异步执行，通常会有分钟级延迟。因此访问入口仍必须检查 `expMinute36`、签名和 `disabledAt`，不能依赖“文档已被删”来保证过期拒绝。 + +## 7. upload 设计 + +upload URL 仍然使用短随机 token： + +```text +/api/system/file/u/ +``` + +原因： + +1. 上传 payload 包含 `maxSize/uploadConstraints/metadata`，URL 内不适合承载。 +2. 上传 URL 是一次上传会话，默认 10 分钟左右，天然适合 TTL session。 +3. upload token 不应按 objectKey 复用，否则可能引入重复 PUT、覆盖对象、策略变更不生效等问题。 + +upload session 文档： + +```ts +export type S3UploadSessionType = { + _id: string; + tokenHash: string; + bucketName: string; + objectKey: string; + maxSize: number; + uploadConstraints: UploadConstraints; + metadata?: Record; + createTime: Date; + expiresAt: Date; + usedAt?: Date; + revokedAt?: Date; +}; +``` + +建议索引： + +```ts +S3UploadSessionSchema.index({ tokenHash: 1 }, { unique: true }); +S3UploadSessionSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 }); +``` + +DB 中只存 hash，不存明文 token： + +```ts +const hashS3UploadToken = (token: string) => + createHmac('sha256', serviceEnv.FILE_TOKEN_KEY).update(token).digest('hex'); +``` + +### 7.1 upload session 文档生命周期 + +| 动作 | 触发时机 | DB 行为 | +|---|---|---| +| 插入 | 调用 `createPresignedPutUrl` 生成上传 URL | 创建 upload session 文档，保存 `tokenHash/bucketName/objectKey/maxSize/uploadConstraints/metadata/expiresAt` | +| 访问 | 前端 PUT 到 `/api/system/file/u/` | 按 `tokenHash` 查询 session，校验过期、撤销、size 和上传策略 | +| 标记使用 | 上传请求通过基础校验或写入完成 | 写入 `usedAt`，用于审计和排查；不依赖它清理 | +| 主动撤销 | 前端取消上传、管理操作或安全事件 | 写入 `revokedAt`，后续请求立即拒绝 | +| 自动清理 | `expiresAt <= now` | Mongo TTL index 自动删除文档 | + +upload session 的清理可以依赖 `expiresAt`，因为它是短生命周期上传会话；即使 TTL monitor 延迟，访问入口也会显式校验 `expiresAt` 和 `revokedAt`。 + +## 8. DB 压力分析 + +### 8.1 签发路径 + +download 签发不再写 per-token 文档： + +```text +文档数 ~= distinct(bucketName, objectKey, filename, responseContentType) +``` + +重复打开页面时，最多命中同一个 alias 文档，并低频推进 `purgeAt`。 + +upload 签发仍然写 session 文档： + +```text +文档数 ~= 10 分钟内真实创建的上传会话数 +``` + +### 8.2 访问路径 + +download 访问： + +1. 过期或篡改 URL：只做本地 HMAC/时间校验，不查 Mongo。 +2. 合法 URL：按 `aliasId` 做一次 indexed lookup。 + +upload 访问： + +1. 按 `tokenHash` 做一次 indexed lookup。 +2. 校验过期、撤销、size、content-type 等上传约束。 + +如果后续线上指标显示 download alias lookup 压力偏高，可以加一层短 TTL read-through cache： + +```text +cache key: s3:download-alias: +cache ttl: min(expiredAt - now, 5 minutes) +``` + +cache 只缓存 alias payload，不缓存整个下载响应。撤销 alias 后最多存在几分钟缓存延迟；如果需要严格撤销，可以在撤销时删除 cache key。 + +## 9. Zod Schema 设计 + +Zod schema 放在 `packages/service/common/s3/accessLink/type.ts`。这些 schema 负责服务内部签发、验证和 API query 校验；实际 API 路由仍需要用 `parseApiInput` 读取 `req.query`，不要直接在 API 文件里写 `Schema.parse(req.query)`。 + +```ts +import z from 'zod'; +import { UploadConstraintsSchema } from '../contracts/type'; + +const UrlSafeTokenSchema = z.string().regex(/^[A-Za-z0-9_-]+$/); +const HexSha256Schema = z.string().length(64).regex(/^[a-f0-9]+$/); + +export const S3AccessBucketNameSchema = z.string().min(1); +export const S3AccessObjectKeySchema = z.string().min(1); + +export const S3DownloadAliasIdSchema = UrlSafeTokenSchema.min(12).max(32); +export const S3DownloadAliasKeySchema = HexSha256Schema; +export const S3DownloadExpiresMinuteSchema = z.string().min(1).max(8).regex(/^[0-9a-z]+$/); +export const S3DownloadSignatureSchema = UrlSafeTokenSchema.min(16).max(64); +export const S3SignedDownloadAliasValueSchema = z + .string() + .regex(/^[A-Za-z0-9_-]{12,32}\.[0-9a-z]{1,8}\.[A-Za-z0-9_-]{16,64}$/); + +export const S3DownloadAliasSchema = z.object({ + aliasId: S3DownloadAliasIdSchema, + aliasKey: S3DownloadAliasKeySchema, + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + filename: z.string().min(1).optional(), + responseContentType: z.string().min(1).optional(), + createTime: z.coerce.date(), + updateTime: z.coerce.date(), + lastIssuedAt: z.coerce.date(), + purgeAt: z.coerce.date(), + disabledAt: z.coerce.date().optional() +}); +export type S3DownloadAliasType = z.infer; + +export const CreateS3DownloadAccessUrlParamsSchema = z.object({ + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + expiredTime: z.coerce.date(), + filename: z.string().min(1).optional(), + responseContentType: z.string().min(1).optional() +}); +export type CreateS3DownloadAccessUrlParams = z.infer< + typeof CreateS3DownloadAccessUrlParamsSchema +>; + +export const ParsedS3SignedDownloadAliasSchema = z.object({ + aliasId: S3DownloadAliasIdSchema, + expMinute36: S3DownloadExpiresMinuteSchema, + sig: S3DownloadSignatureSchema +}); +export type ParsedS3SignedDownloadAlias = z.infer; + +export const S3ProxyDownloadPayloadSchema = z.object({ + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + filename: z.string().min(1).optional(), + responseContentType: z.string().min(1).optional() +}); +export type S3ProxyDownloadPayload = z.infer; +``` + +upload session 相关 schema 同文件定义： + +```ts +export const S3UploadTokenSchema = UrlSafeTokenSchema.min(20).max(64); +export const S3UploadTokenHashSchema = HexSha256Schema; + +export const S3UploadSessionSchema = z.object({ + tokenHash: S3UploadTokenHashSchema, + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + maxSize: z.number().positive(), + uploadConstraints: UploadConstraintsSchema, + metadata: z.record(z.string(), z.string()).optional(), + createTime: z.coerce.date(), + expiresAt: z.coerce.date(), + usedAt: z.coerce.date().optional(), + revokedAt: z.coerce.date().optional() +}); +export type S3UploadSessionType = z.infer; + +export const CreateS3UploadAccessUrlParamsSchema = z.object({ + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + expiredTime: z.coerce.date(), + maxSize: z.number().positive(), + uploadConstraints: UploadConstraintsSchema, + metadata: z.record(z.string(), z.string()).optional() +}); +export type CreateS3UploadAccessUrlParams = z.infer; + +export const S3ProxyUploadPayloadSchema = S3UploadSessionSchema.pick({ + bucketName: true, + objectKey: true, + maxSize: true, + uploadConstraints: true, + metadata: true +}); +export type S3ProxyUploadPayload = z.infer; +``` + +API route query schema 同文件定义，供 `parseApiInput` 使用： + +```ts +export const S3DownloadAccessRouteQuerySchema = z.object({ + signedAlias: S3SignedDownloadAliasValueSchema +}); + +export const S3UploadAccessRouteQuerySchema = z.object({ + token: S3UploadTokenSchema +}); +``` + +## 10. MongoDB Schema 设计 + +### 10.1 download alias Mongo Schema + +`packages/service/common/s3/accessLink/downloadAlias/schema.ts` + +```ts +import { getLogger, LogCategories } from '../../../logger'; +import { getMongoModel, Schema } from '../../../mongo'; +import type { S3DownloadAliasType } from '../type'; + +export const S3DownloadAliasCollectionName = 's3_download_aliases'; + +const logger = getLogger(LogCategories.INFRA.MONGO); + +const S3DownloadAliasMongoSchema = new Schema({ + aliasId: { + type: String, + required: true, + unique: true + }, + aliasKey: { + type: String, + required: true, + unique: true + }, + bucketName: { + type: String, + required: true + }, + objectKey: { + type: String, + required: true + }, + filename: String, + responseContentType: String, + createTime: { + type: Date, + default: () => new Date() + }, + updateTime: { + type: Date, + default: () => new Date() + }, + lastIssuedAt: { + type: Date, + required: true + }, + purgeAt: { + type: Date, + required: true + }, + disabledAt: Date +}); + +try { + S3DownloadAliasMongoSchema.index({ aliasId: 1 }, { unique: true }); + S3DownloadAliasMongoSchema.index({ aliasKey: 1 }, { unique: true }); + S3DownloadAliasMongoSchema.index({ purgeAt: 1 }, { expireAfterSeconds: 0 }); + S3DownloadAliasMongoSchema.index({ bucketName: 1, objectKey: 1 }); +} catch (error) { + logger.error('Failed to build S3 download alias indexes', { error }); +} + +export const MongoS3DownloadAlias = getMongoModel( + S3DownloadAliasCollectionName, + S3DownloadAliasMongoSchema +); +``` + +索引设计： + +| 索引 | 类型 | 用途 | +|---|---|---| +| `{ aliasId: 1 }` | unique | 下载访问时按短 id 查资源 | +| `{ aliasKey: 1 }` | unique | 签发时按资源变体去重，防并发重复创建 | +| `{ purgeAt: 1 }` | TTL | 最后一个已签发 URL 过期后自动清理 alias | +| `{ bucketName: 1, objectKey: 1 }` | normal | S3 对象删除时 best-effort 清理 alias | + +### 10.2 upload session Mongo Schema + +`packages/service/common/s3/accessLink/uploadSession/schema.ts` + +```ts +import { getLogger, LogCategories } from '../../../logger'; +import { getMongoModel, Schema } from '../../../mongo'; +import type { S3UploadSessionType } from '../type'; + +export const S3UploadSessionCollectionName = 's3_upload_sessions'; + +const logger = getLogger(LogCategories.INFRA.MONGO); + +const S3UploadSessionMongoSchema = new Schema({ + tokenHash: { + type: String, + required: true, + unique: true + }, + bucketName: { + type: String, + required: true + }, + objectKey: { + type: String, + required: true + }, + maxSize: { + type: Number, + required: true + }, + uploadConstraints: { + type: Object, + required: true + }, + metadata: Object, + createTime: { + type: Date, + default: () => new Date() + }, + expiresAt: { + type: Date, + required: true + }, + usedAt: Date, + revokedAt: Date +}); + +try { + S3UploadSessionMongoSchema.index({ tokenHash: 1 }, { unique: true }); + S3UploadSessionMongoSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 }); + S3UploadSessionMongoSchema.index({ bucketName: 1, objectKey: 1 }); +} catch (error) { + logger.error('Failed to build S3 upload session indexes', { error }); +} + +export const MongoS3UploadSession = getMongoModel( + S3UploadSessionCollectionName, + S3UploadSessionMongoSchema +); +``` + +索引设计： + +| 索引 | 类型 | 用途 | +|---|---|---| +| `{ tokenHash: 1 }` | unique | 上传访问时按短 token hash 查 session | +| `{ expiresAt: 1 }` | TTL | 上传 session 过期后自动清理 | +| `{ bucketName: 1, objectKey: 1 }` | normal | 排查或对象删除时定位相关 session | + +## 11. 文件组织 + +| 文件 | 类型 | 职责 | +|---|---|---| +| `packages/service/common/s3/accessLink/type.ts` | 新增 | download alias、upload session、签发参数类型 | +| `packages/service/common/s3/accessLink/constants.ts` | 新增 | route path、签名版本、TTL bucket、purge grace | +| `packages/service/common/s3/accessLink/error.ts` | 新增 | 短链内部错误分类和对外错误映射 | +| `packages/service/common/s3/accessLink/utils.ts` | 新增 | base36 时间、HMAC、constant-time compare、URL 构造 | +| `packages/service/common/s3/accessLink/index.ts` | 新增 | 对外统一导出 accessLink 能力 | +| `packages/service/common/s3/accessLink/downloadAlias/schema.ts` | 新增 | `MongoS3DownloadAlias` 和索引 | +| `packages/service/common/s3/accessLink/downloadAlias/entity.ts` | 新增 | alias create/find/update/delete 数据访问 | +| `packages/service/common/s3/accessLink/downloadAlias/service.ts` | 新增 | download 签发、验证、撤销 | +| `packages/service/common/s3/accessLink/uploadSession/schema.ts` | 新增 | `MongoS3UploadSession` 和 TTL 索引 | +| `packages/service/common/s3/accessLink/uploadSession/entity.ts` | 新增 | upload session create/find/markUsed/revoke | +| `packages/service/common/s3/accessLink/uploadSession/service.ts` | 新增 | upload 签发、验证、撤销 | +| `projects/app/src/service/common/s3/proxy.ts` | 新增 | 下载/上传代理公共逻辑 | +| `projects/app/src/pages/api/system/file/d/[signedAlias].ts` | 新增 | 短下载入口 | +| `projects/app/src/pages/api/system/file/u/[token].ts` | 新增 | 短上传入口 | +| `projects/app/src/pages/api/system/file/download/[token].ts` | 修改 | 复用公共下载代理，保留旧 JWT | +| `projects/app/src/pages/api/system/file/upload/[token].ts` | 修改 | 复用公共上传代理，保留旧 JWT | +| `packages/service/common/s3/buckets/base.ts` | 修改 | 新签发默认返回短 URL | +| `packages/service/common/s3/queue/delete.ts` | 修改 | 删除队列执行后 best-effort 清理 download alias | + +## 12. 函数组织 + +### 12.1 `accessLink/utils.ts` + +| 函数 | 职责 | +|---|---| +| `encodeExpiresAtMinute(date)` | Date 转 base36 unix minute | +| `decodeExpiresAtMinute(value)` | base36 unix minute 转 Date | +| `resolveDownloadExpiresAt(expiredTime)` | 按 TTL bucket 向上取整 | +| `signS3DownloadAlias({ aliasId, expMinute36 })` | 生成 HMAC 签名 | +| `parseSignedS3DownloadAlias(value)` | 解析 `aliasId.exp.sig` | +| `assertS3DownloadAliasSignature(value)` | 校验过期时间和签名 | +| `generateS3AliasId()` | 生成 URL-safe alias id | +| `generateS3UploadToken()` | 生成 URL-safe upload token | +| `hashS3UploadToken(token)` | 使用 `FILE_TOKEN_KEY` 派生 upload token hash | +| `buildS3DownloadUrl(signedAlias)` | 生成 `/api/system/file/d/` | +| `buildS3UploadUrl(token)` | 生成 `/api/system/file/u/` | + +### 12.2 `downloadAlias/entity.ts` + +| 函数 | 职责 | +|---|---| +| `findS3DownloadAliasByAliasKey(aliasKey)` | 签发阶段按资源变体查 alias | +| `findS3DownloadAliasByAliasId(aliasId)` | 下载阶段按 aliasId 查 payload | +| `createS3DownloadAlias(data)` | 创建 alias | +| `touchS3DownloadAliasPurgeAt({ aliasKey, purgeAt })` | 用 `$max` 推进清理时间 | +| `disableS3DownloadAliasByAliasId(aliasId)` | 撤销 alias | +| `deleteS3DownloadAliasByObject({ bucketName, objectKey })` | 对象删除时 best-effort 清理 alias | + +### 12.3 `downloadAlias/service.ts` + +| 函数 | 职责 | +|---|---| +| `createS3DownloadAccessUrl(params)` | 创建或复用 alias，并返回 signed alias URL | +| `verifyS3DownloadAccess(value)` | 校验 signed alias 并返回下载 payload | +| `revokeS3DownloadAlias(aliasId)` | 撤销 alias | + +函数注释要求： + +1. `createS3DownloadAccessUrl` 必须说明它只承载已授权后的存储上下文，不做 app/dataset/team 权限判断。 +2. `verifyS3DownloadAccess` 必须说明 URL 的过期由 `expMinute36 + HMAC` 保证，Mongo alias 只负责资源映射。 + +### 12.4 `uploadSession/service.ts` + +| 函数 | 职责 | +|---|---| +| `createS3UploadAccessUrl(params)` | 创建 upload session 并返回短 URL | +| `verifyS3UploadSessionToken(token)` | 校验 upload token 并返回上传 payload | +| `revokeS3UploadSessionToken(token)` | 撤销 upload session | + +### 12.5 `projects/app/src/service/common/s3/proxy.ts` + +| 函数 | 职责 | +|---|---| +| `handleS3ProxyDownload({ req, res, payload })` | 设置 header 并输出下载 stream | +| `handleS3ProxyUpload({ req, payload })` | 校验上传流并写入 S3 | +| `parseS3ProxyContentLength(req)` | 解析 content-length | +| `buildS3UploadMetadata({ metadata, filename })` | 构造上传 metadata | +| `handleS3ProxyRouteError({ res, error })` | 把可预期文件代理错误映射为稳定 HTTP status 和业务错误响应 | + +## 13. API 行为 + +短下载入口： + +```ts +const payload = await verifyS3DownloadAccess(req.query.signedAlias); +return handleS3ProxyDownload({ req, res, payload }); +``` + +短上传入口： + +```ts +const payload = await verifyS3UploadSessionToken(req.query.token); +return handleS3ProxyUpload({ req, payload }); +``` + +旧 JWT 入口： + +1. 继续使用 `jwtVerifyS3DownloadToken` / `jwtVerifyS3UploadToken`。 +2. 验证后调用同一个 proxy handler。 +3. 不再复制 stream/header/upload guard 逻辑。 +4. 不再导出或调用 `jwtSignS3DownloadToken` / `jwtSignS3UploadToken`，避免新代码继续签发旧 JWT。 + +## 14. 错误设计 + +### 14.1 设计原则 + +文件代理入口不能把所有失败都变成 500。PR1 需要把错误分为两类： + +| 类型 | 处理方式 | +|---|---| +| 可预期访问错误 | 映射成稳定业务错误和 4xx HTTP status | +| 系统异常 | 记录 error 日志，返回 500，不泄露 bucket/objectKey/token | + +注意：`NextAPI` 默认 catch 会用 `jsonRes(code: 500, error)` 输出错误；如果短链路由直接把 `CommonErrEnum.unAuthFile` 抛到默认 catch，业务 `statusText` 可以保留，但 HTTP status 可能仍是 500。因此 `d/[signedAlias].ts`、`u/[token].ts`、旧 `download/[token].ts` 和旧 `upload/[token].ts` 都应该在路由层捕获文件代理可预期错误，并调用 `handleS3ProxyRouteError` 写出响应。 + +### 14.2 内部错误分类 + +`packages/service/common/s3/accessLink/error.ts` + +```ts +export enum S3AccessLinkErrCode { + invalidSignedAlias = 'InvalidSignedAlias', + expiredSignedAlias = 'ExpiredSignedAlias', + invalidSignedAliasSignature = 'InvalidSignedAliasSignature', + downloadAliasNotFound = 'DownloadAliasNotFound', + downloadAliasRevoked = 'DownloadAliasRevoked', + uploadSessionNotFound = 'UploadSessionNotFound', + uploadSessionExpired = 'UploadSessionExpired', + uploadSessionRevoked = 'UploadSessionRevoked' +} + +export class S3AccessLinkError extends Error { + constructor(public readonly code: S3AccessLinkErrCode) { + super(code); + this.name = 'S3AccessLinkError'; + } +} +``` + +内部错误只用于 service/entity/utils 之间传递分类，不直接作为 API 响应暴露给客户端。这样可以避免把“alias 不存在”和“签名错”暴露成可枚举资源存在性的信号。 + +### 14.3 对外错误映射 + +| 场景 | 内部错误或来源 | 对外错误 | HTTP status | 说明 | +|---|---|---|---:|---| +| signed alias 格式错误 | `invalidSignedAlias` | `CommonErrEnum.unAuthFile` | 403 | 不告诉调用方是格式错还是权限错 | +| signed alias 已过期 | `expiredSignedAlias` | `CommonErrEnum.unAuthFile` | 403 | 与旧 JWT 过期语义一致 | +| signed alias 签名错误 | `invalidSignedAliasSignature` | `CommonErrEnum.unAuthFile` | 403 | 防止篡改和枚举 | +| alias 不存在 | `downloadAliasNotFound` | `CommonErrEnum.unAuthFile` | 403 | 不泄露资源存在性 | +| alias 被撤销 | `downloadAliasRevoked` | `CommonErrEnum.unAuthFile` | 403 | 与无权访问一致 | +| S3 对象不存在 | `bucket.getFileStream` 返回空 | `CommonErrEnum.fileNotFound` | 404 | alias 合法但对象已不存在 | +| bucket 配置不存在 | `global.s3BucketMap[bucketName]` 空 | `Error('S3 bucket not found')` | 500 | 服务端配置问题，不暴露 bucketName | +| upload token 不存在 | `uploadSessionNotFound` | `CommonErrEnum.unAuthFile` | 403 | 不泄露 session 状态 | +| upload token 过期 | `uploadSessionExpired` | `CommonErrEnum.unAuthFile` | 403 | 与旧 JWT 过期语义一致 | +| upload token 被撤销 | `uploadSessionRevoked` | `CommonErrEnum.unAuthFile` | 403 | 前端 abort 后继续上传应失败 | +| 上传体超限 | `EntityTooLarge` | `EntityTooLarge` | 413 | 保持现有前端 `parseS3UploadError` 可识别 | +| 上传类型不允许 | 现有上传校验错误 | 原错误透传给 `jsonRes` | 400 | 沿用现有上传校验错误，不在 PR1 新增短链错误码 | +| 上传类型不匹配 | 现有上传校验错误 | 原错误透传给 `jsonRes` | 400 | 沿用现有上传校验错误，不在 PR1 新增短链错误码 | +| API query 校验失败 | `parseApiInput` | Zod request error | 400 | 由 `NextAPI` 的 Zod 降噪逻辑处理 | +| Method 不允许 | 路由方法校验 | `Method not allowed` | 405 | 直接 `jsonRes(code: 405, error)` 或设置 405 | + +### 14.4 route 层错误响应 helper + +`projects/app/src/service/common/s3/proxy.ts` + +```ts +export function resolveS3ProxyErrorResponse(error: unknown): { + httpStatus: number; + publicError: unknown; +} { + if (error instanceof S3AccessLinkError) { + return { + httpStatus: 403, + publicError: CommonErrEnum.unAuthFile + }; + } + + if (error === CommonErrEnum.fileNotFound) { + return { + httpStatus: 404, + publicError: CommonErrEnum.fileNotFound + }; + } + + if (error instanceof Error && error.message === 'EntityTooLarge') { + return { + httpStatus: 413, + publicError: error + }; + } + + return { + httpStatus: 500, + publicError: error + }; +} + +export function handleS3ProxyRouteError({ + res, + error +}: { + res: NextApiResponse; + error: unknown; +}) { + const { httpStatus, publicError } = resolveS3ProxyErrorResponse(error); + + return jsonRes(res, { + code: httpStatus, + error: publicError + }); +} +``` + +这里 `code: httpStatus` 的作用是确保短链自有错误最终 HTTP status 正确；业务响应体里的 `code/statusText/message` 仍由 `ERROR_RESPONSE` 或现有错误解析逻辑生成。 + +上传文件类型错误继续复用现有上传校验链路：`validateUploadFile` 抛出的业务错误已经在全局 `ERROR_RESPONSE/jsonRes` 中有映射，PR1 不需要在短链模块里 import 或匹配 `S3ErrEnum`。这样可以避免把“短链鉴权错误”和“上传策略错误”耦合在一起；真正整理上传错误枚举应放到 PR2 的上传策略重构里处理。 + +### 14.5 抛错边界 + +| 模块 | 可以抛出的错误 | 不能做的事 | +|---|---|---| +| `accessLink/utils.ts` | `S3AccessLinkError` | 不返回 `undefined` 让上层猜原因 | +| `downloadAlias/service.ts` | `S3AccessLinkError`、系统异常 | 不直接 import `jsonRes` | +| `uploadSession/service.ts` | `S3AccessLinkError`、系统异常 | 不直接 import `jsonRes` | +| `projects/app/src/service/common/s3/proxy.ts` | `CommonErrEnum.fileNotFound`、`EntityTooLarge`、现有上传校验业务错误、系统异常 | 不吞 stream/upload 错误 | +| API route | 捕获错误并调用 `handleS3ProxyRouteError` | 不把可预期代理错误落到默认 500 | + +### 14.6 测试要求 + +PR1 必须补充错误响应测试： + +1. 过期 signed alias 返回 HTTP 403，`statusText=unAuthFile`。 +2. 篡改 `expMinute36` 返回 HTTP 403，`statusText=unAuthFile`。 +3. alias 不存在返回 HTTP 403，`statusText=unAuthFile`。 +4. alias 合法但 S3 对象不存在返回 HTTP 404，`statusText=fileNotFound`。 +5. upload token 不存在、过期、撤销均返回 HTTP 403，`statusText=unAuthFile`。 +6. 上传超过 `maxSize` 返回 HTTP 413，并能被 `parseS3UploadError` 识别为文件过大。 +7. 上传类型不允许返回 HTTP 400，并能被 `parseS3UploadError` 识别为文件类型错误。 +8. 未知异常返回 HTTP 500，日志中不能输出明文 token。 + +## 15. 文件影响清单 + +### 15.1 新增文件 + +| 文件 | 原因 | +|---|---| +| `packages/service/common/s3/accessLink/type.ts` | Zod schema、类型推导、API query schema | +| `packages/service/common/s3/accessLink/constants.ts` | 路由常量、签名版本、TTL bucket、purge grace | +| `packages/service/common/s3/accessLink/error.ts` | 短链内部错误分类和对外映射辅助 | +| `packages/service/common/s3/accessLink/utils.ts` | HMAC 签名、base36 过期时间、短 URL 构造、token hash | +| `packages/service/common/s3/accessLink/index.ts` | 统一导出，避免外部引用子文件路径 | +| `packages/service/common/s3/accessLink/downloadAlias/schema.ts` | download alias Mongo model 和索引 | +| `packages/service/common/s3/accessLink/downloadAlias/entity.ts` | download alias 数据访问 | +| `packages/service/common/s3/accessLink/downloadAlias/service.ts` | download alias 签发、校验、撤销 | +| `packages/service/common/s3/accessLink/uploadSession/schema.ts` | upload session Mongo model 和 TTL 索引 | +| `packages/service/common/s3/accessLink/uploadSession/entity.ts` | upload session 数据访问 | +| `packages/service/common/s3/accessLink/uploadSession/service.ts` | upload session 签发、校验、撤销 | +| `projects/app/src/service/common/s3/proxy.ts` | 复用下载和上传代理逻辑，并统一代理错误响应 | +| `projects/app/src/pages/api/system/file/d/[signedAlias].ts` | 新短下载入口 | +| `projects/app/src/pages/api/system/file/u/[token].ts` | 新短上传入口 | +| `packages/service/test/common/s3/accessLink.test.ts` | download alias 和 upload session 单元测试 | +| `projects/app/test/api/system/file/accessLink.test.ts` | 新短路由和旧 JWT 兼容测试 | + +### 15.2 修改文件 + +| 文件 | 修改点 | +|---|---| +| `packages/service/common/s3/buckets/base.ts` | `createExternalUrl(proxy)` 改为签发短 download alias URL；`createPresignedPutUrl` 改为签发短 upload session URL；`removeObject` 直接删除对象后 best-effort 删除相关 download alias | +| `packages/service/common/s3/security/token.ts` | 移除旧 JWT 签发函数，只保留旧 JWT 下载/上传校验函数，用于兼容已签发旧链接 | +| `projects/app/src/pages/api/system/file/download/[token].ts` | 旧 JWT 下载入口复用 `handleS3ProxyDownload` | +| `projects/app/src/pages/api/system/file/upload/[token].ts` | 旧 JWT 上传入口复用 `handleS3ProxyUpload` | +| `packages/service/common/s3/queue/delete.ts` | 删除对象成功后 best-effort 删除相关 download alias | + +### 15.3 不修改文件 + +| 文件 | 原因 | +|---|---| +| `packages/service/common/s3/validation/upload.ts` | 文件类型校验属于 PR2 | +| ChatBox 上传组件相关文件 | abort 行为属于 PR3 | + +## 16. Tasks + +- [x] PR1-T1 新增 `accessLink/type.ts`，定义 download alias、upload session 和签发参数类型。 +- [x] PR1-T2 新增 `accessLink/constants.ts`，定义短链 route、签名版本、TTL bucket 和 purge grace。 +- [x] PR1-T3 新增 `accessLink/error.ts`，定义 `S3AccessLinkError`、错误码和代理错误映射约定。 +- [x] PR1-T4 新增 `accessLink/utils.ts`，实现 base36 时间、HMAC 签名、签名校验、短 URL 构造。 +- [x] PR1-T5 新增 `downloadAlias/schema.ts`，建立 `aliasId` 唯一索引、`aliasKey` 唯一索引和 `purgeAt` TTL 索引。 +- [x] PR1-T6 新增 `downloadAlias/entity.ts`，封装 alias create/find/touch/disable/delete。 +- [x] PR1-T7 新增 `downloadAlias/service.ts`，实现 `createS3DownloadAccessUrl` 和 `verifyS3DownloadAccess`。 +- [x] PR1-T8 新增 `uploadSession/schema.ts`，建立 `tokenHash` 唯一索引和 `expiresAt` TTL 索引。 +- [x] PR1-T9 新增 `uploadSession/entity.ts`，封装 upload session create/find/markUsed/revoke。 +- [x] PR1-T10 新增 `uploadSession/service.ts`，实现 `createS3UploadAccessUrl` 与 `verifyS3UploadSessionToken`。 +- [x] PR1-T11 新增 `accessLink/index.ts`，统一导出短链模块能力。 +- [x] PR1-T12 抽取 `projects/app/src/service/common/s3/proxy.ts`，迁移现有下载代理公共逻辑。 +- [x] PR1-T13 在同一个 proxy 文件中迁移现有上传代理公共逻辑，并实现 `handleS3ProxyRouteError`。 +- [x] PR1-T14 新增 `d/[signedAlias].ts`，短下载入口用 `parseApiInput` 校验 query 后调用 signed alias 校验和代理。 +- [x] PR1-T15 新增 `u/[token].ts`，短上传入口用 `parseApiInput` 校验 query 后调用 upload session 校验和代理。 +- [x] PR1-T16 改造旧 `download/[token].ts` 和 `upload/[token].ts`，保留 JWT 兼容并复用 proxy。 +- [x] PR1-T17 改造 `S3BaseBucket.createExternalUrl` proxy 分支，默认签发短 download alias URL。 +- [x] PR1-T18 改造 `S3BaseBucket.createPresignedPutUrl`，默认签发短 upload session URL，并让 `previewUrl` 走短 download alias URL。 +- [x] PR1-T19 在 `S3BaseBucket.removeObject` 和 S3 删除队列中 best-effort 接入 `deleteS3DownloadAliasByObject`。 +- [x] PR1-T20 补充 signed alias 签发、复用、过期、篡改、撤销、purgeAt 推进和错误映射测试。 +- [x] PR1-T21 补充 upload session 签发、校验、过期、撤销、purpose 隔离和错误映射测试。 +- [x] PR1-T22 补充短下载代理、短上传代理和旧 JWT 兼容测试。 +- [x] PR1-T23 运行 PR1 局部测试。 + +## 17. 测试 + +建议局部测试： + +```bash +pnpm test packages/service/test/common/s3/token.test.ts +pnpm test projects/app/test/api/system/file/sourceContentType.test.ts +``` + +新增或扩展用例： + +1. download 短 URL 不包含 JWT payload 和 objectKey。 +2. 同一 objectKey 重复签发 download URL，只复用同一个 alias 文档。 +3. download URL 修改 `expMinute36` 后被拒绝。 +4. download URL 修改 `sig` 后被拒绝。 +5. 过期 download URL 在查 Mongo 前被拒绝。 +6. revoked alias 被拒绝。 +7. 签发时 `purgeAt` 至少晚于 URL 过期时间。 +8. upload 票据不能用于 download。 +9. 过期 upload session 被拒绝。 +10. DB 中只存 upload token hash，不存 upload token 明文。 +11. 旧 JWT 下载和上传仍可用。 +12. 可预期短链和上传错误不会被默认包装成 HTTP 500。 + +## 18. 验收标准 + +1. 新签发的 proxy 下载 URL 形如 `/api/system/file/d/..`。 +2. 新签发的 proxy 上传 URL 形如 `/api/system/file/u/`。 +3. 下载短 URL 长度明显短于旧 JWT URL，且不包含 objectKey。 +4. 反复打开同一页面不会为同一图片持续新增 token 文档。 +5. download alias 文档可通过 `purgeAt` TTL 自动清理。 +6. upload session 文档可通过 `expiresAt` TTL 自动清理。 +7. 旧 JWT 路由仍可用。 +8. 下载和上传代理行为与改造前一致。 +9. 过期、篡改、撤销、超限、文件不存在等可预期错误能被上游捕获，并返回稳定 `statusText` 和 HTTP status。 diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts index a9df795458c5..083943ebed9f 100644 --- a/packages/global/common/string/tools.ts +++ b/packages/global/common/string/tools.ts @@ -109,7 +109,7 @@ export const sliceStrStartEnd = (str: string | null = '', start: number, end: nu */ export const parseFileExtensionFromUrl = (url = '') => { // Prefer explicit filename in query params for proxy links: - // e.g. /api/system/file/download/?filename=image.jpg + // e.g. /api/system/file/d/, or a legacy proxy URL carrying filename in query. try { const parsedUrl = new URL(url, 'http://localhost'); const queryFilename = diff --git a/packages/global/openapi/core/dataset/data/api.ts b/packages/global/openapi/core/dataset/data/api.ts index 9843e2ce5a37..3f16dada845c 100644 --- a/packages/global/openapi/core/dataset/data/api.ts +++ b/packages/global/openapi/core/dataset/data/api.ts @@ -59,7 +59,7 @@ export const UpdateDatasetDataBodySchema = UpdateDatasetDataPropsSchema; export type UpdateDatasetDataBody = z.infer; export const UpdateDatasetDataResponseSchema = z.object({ q: z.string().optional().meta({ - example: '![image.png](/api/system/file/download/xxx?filename=image.png)', + example: '![image.png](/api/system/file/d/alias.exp.sig)', description: '展示态问题/主文本，内部 S3 图片会替换为签名访问地址' }), a: z.string().optional().meta({ diff --git a/packages/service/common/s3/accessLink/constants.ts b/packages/service/common/s3/accessLink/constants.ts new file mode 100644 index 000000000000..6e47e96d8276 --- /dev/null +++ b/packages/service/common/s3/accessLink/constants.ts @@ -0,0 +1,21 @@ +export const S3_ACCESS_LINK_ROUTES = { + download: '/api/system/file/d', + upload: '/api/system/file/u' +} as const; + +export const S3_DOWNLOAD_ALIAS_SIGN_VERSION = 'v1'; +export const S3_DOWNLOAD_ALIAS_ID_LENGTH = 16; +export const S3_DOWNLOAD_SIGNATURE_LENGTH = 22; +export const S3_UPLOAD_TOKEN_LENGTH = 22; +export const S3_ACCESS_LINK_PURGE_GRACE_HOURS = 24; + +export const S3_DOWNLOAD_EXPIRE_BUCKET_MS = { + short: 15 * 60 * 1000, + medium: 60 * 60 * 1000, + long: 24 * 60 * 60 * 1000 +} as const; + +export const S3_DOWNLOAD_EXPIRE_BUCKET_THRESHOLD_MS = { + short: 2 * 60 * 60 * 1000, + medium: 24 * 60 * 60 * 1000 +} as const; diff --git a/packages/service/common/s3/accessLink/downloadAlias/entity.ts b/packages/service/common/s3/accessLink/downloadAlias/entity.ts new file mode 100644 index 000000000000..ed76f7507d6a --- /dev/null +++ b/packages/service/common/s3/accessLink/downloadAlias/entity.ts @@ -0,0 +1,102 @@ +import { MongoS3DownloadAlias } from './schema'; +import type { S3DownloadAliasType } from '../type'; + +type CreateS3DownloadAliasData = Omit & { + createTime?: Date; + updateTime?: Date; +}; + +const isMongoDuplicateKeyError = (error: unknown) => + !!error && typeof error === 'object' && 'code' in error && error.code === 11000; + +export const findS3DownloadAliasByAliasKey = (aliasKey: string) => { + return MongoS3DownloadAlias.findOne({ aliasKey }).lean(); +}; + +export const findS3DownloadAliasByAliasId = (aliasId: string) => { + return MongoS3DownloadAlias.findOne({ aliasId }).lean(); +}; + +export const createS3DownloadAlias = async (data: CreateS3DownloadAliasData) => { + try { + const now = new Date(); + const [created] = await MongoS3DownloadAlias.create([ + { + ...data, + createTime: data.createTime ?? now, + updateTime: data.updateTime ?? now + } + ]); + + return created.toObject() as S3DownloadAliasType; + } catch (error) { + if (isMongoDuplicateKeyError(error)) { + return findS3DownloadAliasByAliasKey(data.aliasKey); + } + + throw error; + } +}; + +export const touchS3DownloadAliasPurgeAt = ({ + aliasKey, + purgeAt, + now = new Date() +}: { + aliasKey: string; + purgeAt: Date; + now?: Date; +}) => { + return MongoS3DownloadAlias.updateOne( + { aliasKey }, + { + $set: { + updateTime: now, + lastIssuedAt: now + }, + $max: { + purgeAt + } + } + ); +}; + +export const disableS3DownloadAliasByAliasId = (aliasId: string) => { + return MongoS3DownloadAlias.updateOne( + { aliasId }, + { + $set: { + disabledAt: new Date(), + updateTime: new Date() + } + } + ); +}; + +export const deleteS3DownloadAliasByObject = ({ + bucketName, + objectKey +}: { + bucketName: string; + objectKey: string; +}) => { + return MongoS3DownloadAlias.deleteMany({ + bucketName, + objectKey + }); +}; + +export const deleteS3DownloadAliasByObjects = ({ + bucketName, + objectKeys +}: { + bucketName: string; + objectKeys: string[]; +}) => { + if (objectKeys.length === 0) return Promise.resolve(); + + return MongoS3DownloadAlias.deleteMany({ + bucketName, + objectKey: { $in: objectKeys } + }); +}; diff --git a/packages/service/common/s3/accessLink/downloadAlias/schema.ts b/packages/service/common/s3/accessLink/downloadAlias/schema.ts new file mode 100644 index 000000000000..548e310837ce --- /dev/null +++ b/packages/service/common/s3/accessLink/downloadAlias/schema.ts @@ -0,0 +1,59 @@ +import { getLogger, LogCategories } from '../../../logger'; +import { getMongoModel, Schema } from '../../../mongo'; +import type { S3DownloadAliasType } from '../type'; + +export const S3DownloadAliasCollectionName = 's3_download_aliases'; + +const logger = getLogger(LogCategories.INFRA.MONGO); + +const S3DownloadAliasMongoSchema = new Schema({ + aliasId: { + type: String, + required: true + }, + aliasKey: { + type: String, + required: true + }, + bucketName: { + type: String, + required: true + }, + objectKey: { + type: String, + required: true + }, + filename: String, + responseContentType: String, + createTime: { + type: Date, + default: () => new Date() + }, + updateTime: { + type: Date, + default: () => new Date() + }, + lastIssuedAt: { + type: Date, + required: true + }, + purgeAt: { + type: Date, + required: true + }, + disabledAt: Date +}); + +try { + S3DownloadAliasMongoSchema.index({ aliasId: 1 }, { unique: true }); + S3DownloadAliasMongoSchema.index({ aliasKey: 1 }, { unique: true }); + S3DownloadAliasMongoSchema.index({ purgeAt: 1 }, { expireAfterSeconds: 0 }); + S3DownloadAliasMongoSchema.index({ bucketName: 1, objectKey: 1 }); +} catch (error) { + logger.error('Failed to build S3 download alias indexes', { error }); +} + +export const MongoS3DownloadAlias = getMongoModel( + S3DownloadAliasCollectionName, + S3DownloadAliasMongoSchema +); diff --git a/packages/service/common/s3/accessLink/downloadAlias/service.ts b/packages/service/common/s3/accessLink/downloadAlias/service.ts new file mode 100644 index 000000000000..6110fec1623a --- /dev/null +++ b/packages/service/common/s3/accessLink/downloadAlias/service.ts @@ -0,0 +1,102 @@ +import { addHours } from 'date-fns'; +import { + CreateS3DownloadAccessUrlParamsSchema, + S3ProxyDownloadPayloadSchema, + type S3ProxyDownloadPayload +} from '../type'; +import { + assertS3DownloadAliasSignature, + buildS3DownloadAliasKey, + buildS3DownloadUrl, + encodeExpiresAtMinute, + generateS3AliasId, + resolveDownloadExpiresAt, + signS3DownloadAlias +} from '../utils'; +import { S3AccessLinkErrCode, S3AccessLinkError } from '../error'; +import { S3_ACCESS_LINK_PURGE_GRACE_HOURS } from '../constants'; +import { + createS3DownloadAlias, + disableS3DownloadAliasByAliasId, + findS3DownloadAliasByAliasId, + findS3DownloadAliasByAliasKey, + touchS3DownloadAliasPurgeAt +} from './entity'; + +/** + * 创建或复用下载 alias，并返回带过期时间与 HMAC 签名的短 URL。 + * + * 该函数只承载已经完成业务鉴权后的存储上下文，不做 app/dataset/team 等归属校验。 + * 调用方必须在传入 objectKey 前确认当前用户有权访问该文件。 + */ +export const createS3DownloadAccessUrl = async (params: unknown) => { + const parsed = CreateS3DownloadAccessUrlParamsSchema.parse(params); + const now = new Date(); + const expiresAt = resolveDownloadExpiresAt(parsed.expiredTime, now); + const expMinute36 = encodeExpiresAtMinute(expiresAt); + const purgeAt = addHours(expiresAt, S3_ACCESS_LINK_PURGE_GRACE_HOURS); + const aliasKey = buildS3DownloadAliasKey(parsed); + + const existingAlias = await findS3DownloadAliasByAliasKey(aliasKey); + const alias = + existingAlias ?? + (await createS3DownloadAlias({ + aliasId: generateS3AliasId(), + aliasKey, + bucketName: parsed.bucketName, + objectKey: parsed.objectKey, + filename: parsed.filename, + responseContentType: parsed.responseContentType, + lastIssuedAt: now, + purgeAt + })); + + if (!alias) { + throw new S3AccessLinkError(S3AccessLinkErrCode.downloadAliasNotFound); + } + + await touchS3DownloadAliasPurgeAt({ + aliasKey, + purgeAt, + now + }); + + const sig = signS3DownloadAlias({ + aliasId: alias.aliasId, + expMinute36 + }); + + return buildS3DownloadUrl(`${alias.aliasId}.${expMinute36}.${sig}`); +}; + +/** + * 校验 signed alias 并返回文件代理下载 payload。 + * + * URL 的过期由 `expMinute36 + HMAC` 保证；Mongo alias 只负责把短 id 映射回真实 + * `bucketName/objectKey`。 + */ +export const verifyS3DownloadAccess = async ( + signedAlias: string +): Promise => { + const { aliasId } = assertS3DownloadAliasSignature(signedAlias); + const alias = await findS3DownloadAliasByAliasId(aliasId); + + if (!alias) { + throw new S3AccessLinkError(S3AccessLinkErrCode.downloadAliasNotFound); + } + + if (alias.disabledAt) { + throw new S3AccessLinkError(S3AccessLinkErrCode.downloadAliasRevoked); + } + + return S3ProxyDownloadPayloadSchema.parse({ + bucketName: alias.bucketName, + objectKey: alias.objectKey, + filename: alias.filename, + responseContentType: alias.responseContentType + }); +}; + +export const revokeS3DownloadAlias = (aliasId: string) => { + return disableS3DownloadAliasByAliasId(aliasId); +}; diff --git a/packages/service/common/s3/accessLink/error.ts b/packages/service/common/s3/accessLink/error.ts new file mode 100644 index 000000000000..aafde9954069 --- /dev/null +++ b/packages/service/common/s3/accessLink/error.ts @@ -0,0 +1,26 @@ +export enum S3AccessLinkErrCode { + invalidSignedAlias = 'InvalidSignedAlias', + expiredSignedAlias = 'ExpiredSignedAlias', + invalidSignedAliasSignature = 'InvalidSignedAliasSignature', + downloadAliasNotFound = 'DownloadAliasNotFound', + downloadAliasRevoked = 'DownloadAliasRevoked', + uploadSessionNotFound = 'UploadSessionNotFound', + uploadSessionExpired = 'UploadSessionExpired', + uploadSessionRevoked = 'UploadSessionRevoked' +} + +/** + * 短链内部错误分类。 + * + * 该错误只在 accessLink service/utils 与文件代理边界之间传递，不直接暴露给客户端。 + * API 层统一把它映射为 `unAuthFile`，避免泄露 alias 是否存在、token 是否过期等枚举信号。 + */ +export class S3AccessLinkError extends Error { + constructor(public readonly code: S3AccessLinkErrCode) { + super(code); + this.name = 'S3AccessLinkError'; + } +} + +export const isS3AccessLinkError = (error: unknown): error is S3AccessLinkError => + error instanceof S3AccessLinkError; diff --git a/packages/service/common/s3/accessLink/index.ts b/packages/service/common/s3/accessLink/index.ts new file mode 100644 index 000000000000..4b434a0ac323 --- /dev/null +++ b/packages/service/common/s3/accessLink/index.ts @@ -0,0 +1,7 @@ +export * from './constants'; +export * from './error'; +export * from './type'; +export * from './utils'; +export * from './downloadAlias/entity'; +export * from './downloadAlias/service'; +export * from './uploadSession/service'; diff --git a/packages/service/common/s3/accessLink/type.ts b/packages/service/common/s3/accessLink/type.ts new file mode 100644 index 000000000000..580cb303ca70 --- /dev/null +++ b/packages/service/common/s3/accessLink/type.ts @@ -0,0 +1,106 @@ +import z from 'zod'; +import { UploadConstraintsSchema } from '../contracts/type'; + +const UrlSafeTokenSchema = z.string().regex(/^[A-Za-z0-9_-]+$/); +const HexSha256Schema = z + .string() + .length(64) + .regex(/^[a-f0-9]+$/); + +export const S3AccessBucketNameSchema = z.string().min(1); +export const S3AccessObjectKeySchema = z.string().min(1); + +export const S3DownloadAliasIdSchema = UrlSafeTokenSchema.min(12).max(32); +export const S3DownloadAliasKeySchema = HexSha256Schema; +export const S3DownloadExpiresMinuteSchema = z + .string() + .min(1) + .max(8) + .regex(/^[0-9a-z]+$/); +export const S3DownloadSignatureSchema = UrlSafeTokenSchema.min(16).max(64); +export const S3SignedDownloadAliasValueSchema = z + .string() + .regex(/^[A-Za-z0-9_-]{12,32}\.[0-9a-z]{1,8}\.[A-Za-z0-9_-]{16,64}$/); + +export const S3DownloadAliasSchema = z.object({ + aliasId: S3DownloadAliasIdSchema, + aliasKey: S3DownloadAliasKeySchema, + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + filename: z.string().min(1).optional(), + responseContentType: z.string().min(1).optional(), + createTime: z.coerce.date(), + updateTime: z.coerce.date(), + lastIssuedAt: z.coerce.date(), + purgeAt: z.coerce.date(), + disabledAt: z.coerce.date().optional() +}); +export type S3DownloadAliasType = z.infer; + +export const CreateS3DownloadAccessUrlParamsSchema = z.object({ + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + expiredTime: z.coerce.date(), + filename: z.string().min(1).optional(), + responseContentType: z.string().min(1).optional() +}); +export type CreateS3DownloadAccessUrlParams = z.infer; + +export const ParsedS3SignedDownloadAliasSchema = z.object({ + aliasId: S3DownloadAliasIdSchema, + expMinute36: S3DownloadExpiresMinuteSchema, + sig: S3DownloadSignatureSchema +}); +export type ParsedS3SignedDownloadAlias = z.infer; + +export const S3ProxyDownloadPayloadSchema = z.object({ + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + filename: z.string().min(1).optional(), + responseContentType: z.string().min(1).optional() +}); +export type S3ProxyDownloadPayload = z.infer; + +export const S3UploadTokenSchema = UrlSafeTokenSchema.min(20).max(64); +export const S3UploadTokenHashSchema = HexSha256Schema; + +export const S3UploadSessionSchema = z.object({ + tokenHash: S3UploadTokenHashSchema, + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + maxSize: z.number().positive(), + uploadConstraints: UploadConstraintsSchema, + metadata: z.record(z.string(), z.string()).optional(), + createTime: z.coerce.date(), + expiresAt: z.coerce.date(), + usedAt: z.coerce.date().optional(), + revokedAt: z.coerce.date().optional() +}); +export type S3UploadSessionType = z.infer; + +export const CreateS3UploadAccessUrlParamsSchema = z.object({ + bucketName: S3AccessBucketNameSchema, + objectKey: S3AccessObjectKeySchema, + expiredTime: z.coerce.date(), + maxSize: z.number().positive(), + uploadConstraints: UploadConstraintsSchema, + metadata: z.record(z.string(), z.string()).optional() +}); +export type CreateS3UploadAccessUrlParams = z.infer; + +export const S3ProxyUploadPayloadSchema = S3UploadSessionSchema.pick({ + bucketName: true, + objectKey: true, + maxSize: true, + uploadConstraints: true, + metadata: true +}); +export type S3ProxyUploadPayload = z.infer; + +export const S3DownloadAccessRouteQuerySchema = z.object({ + signedAlias: S3SignedDownloadAliasValueSchema +}); + +export const S3UploadAccessRouteQuerySchema = z.object({ + token: S3UploadTokenSchema +}); diff --git a/packages/service/common/s3/accessLink/uploadSession/entity.ts b/packages/service/common/s3/accessLink/uploadSession/entity.ts new file mode 100644 index 000000000000..979b3959f33c --- /dev/null +++ b/packages/service/common/s3/accessLink/uploadSession/entity.ts @@ -0,0 +1,43 @@ +import { MongoS3UploadSession } from './schema'; +import type { S3UploadSessionType } from '../type'; + +type CreateS3UploadSessionData = Omit & { + createTime?: Date; +}; + +export const createS3UploadSession = async (data: CreateS3UploadSessionData) => { + const [created] = await MongoS3UploadSession.create([ + { + ...data, + createTime: data.createTime ?? new Date() + } + ]); + + return created.toObject() as S3UploadSessionType; +}; + +export const findS3UploadSessionByTokenHash = (tokenHash: string) => { + return MongoS3UploadSession.findOne({ tokenHash }).lean(); +}; + +export const markS3UploadSessionUsed = (tokenHash: string) => { + return MongoS3UploadSession.updateOne( + { tokenHash }, + { + $set: { + usedAt: new Date() + } + } + ); +}; + +export const revokeS3UploadSessionByTokenHash = (tokenHash: string) => { + return MongoS3UploadSession.updateOne( + { tokenHash }, + { + $set: { + revokedAt: new Date() + } + } + ); +}; diff --git a/packages/service/common/s3/accessLink/uploadSession/schema.ts b/packages/service/common/s3/accessLink/uploadSession/schema.ts new file mode 100644 index 000000000000..7803f577bd6d --- /dev/null +++ b/packages/service/common/s3/accessLink/uploadSession/schema.ts @@ -0,0 +1,54 @@ +import { getLogger, LogCategories } from '../../../logger'; +import { getMongoModel, Schema } from '../../../mongo'; +import type { S3UploadSessionType } from '../type'; + +export const S3UploadSessionCollectionName = 's3_upload_sessions'; + +const logger = getLogger(LogCategories.INFRA.MONGO); + +const S3UploadSessionMongoSchema = new Schema({ + tokenHash: { + type: String, + required: true + }, + bucketName: { + type: String, + required: true + }, + objectKey: { + type: String, + required: true + }, + maxSize: { + type: Number, + required: true + }, + uploadConstraints: { + type: Object, + required: true + }, + metadata: Object, + createTime: { + type: Date, + default: () => new Date() + }, + expiresAt: { + type: Date, + required: true + }, + usedAt: Date, + revokedAt: Date +}); + +try { + S3UploadSessionMongoSchema.index({ tokenHash: 1 }, { unique: true }); + S3UploadSessionMongoSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 }); + S3UploadSessionMongoSchema.index({ bucketName: 1, objectKey: 1 }); +} catch (error) { + logger.error('Failed to build S3 upload session indexes', { error }); +} + +export const MongoS3UploadSession = getMongoModel( + S3UploadSessionCollectionName, + S3UploadSessionMongoSchema +); diff --git a/packages/service/common/s3/accessLink/uploadSession/service.ts b/packages/service/common/s3/accessLink/uploadSession/service.ts new file mode 100644 index 000000000000..73dfb9019cdf --- /dev/null +++ b/packages/service/common/s3/accessLink/uploadSession/service.ts @@ -0,0 +1,78 @@ +import { + CreateS3UploadAccessUrlParamsSchema, + S3ProxyUploadPayloadSchema, + S3UploadTokenSchema, + type S3ProxyUploadPayload +} from '../type'; +import { buildS3UploadUrl, generateS3UploadToken, hashS3UploadToken } from '../utils'; +import { S3AccessLinkErrCode, S3AccessLinkError } from '../error'; +import { + createS3UploadSession, + findS3UploadSessionByTokenHash, + markS3UploadSessionUsed, + revokeS3UploadSessionByTokenHash +} from './entity'; + +/** + * 创建一次上传会话并返回短上传 URL。 + * + * 上传 session 承载 maxSize/uploadConstraints/metadata 等服务端策略，不按 objectKey 复用， + * 避免重复 PUT、覆盖对象和策略变更不生效。 + */ +export const createS3UploadAccessUrl = async (params: unknown) => { + const parsed = CreateS3UploadAccessUrlParamsSchema.parse(params); + const token = generateS3UploadToken(); + const tokenHash = hashS3UploadToken(token); + + await createS3UploadSession({ + tokenHash, + bucketName: parsed.bucketName, + objectKey: parsed.objectKey, + maxSize: parsed.maxSize, + uploadConstraints: parsed.uploadConstraints, + metadata: parsed.metadata, + expiresAt: parsed.expiredTime + }); + + return buildS3UploadUrl(token); +}; + +export const verifyS3UploadSessionToken = async ( + token: string, + now = new Date() +): Promise => { + const parsedToken = S3UploadTokenSchema.safeParse(token); + + if (!parsedToken.success) { + throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionNotFound); + } + + const tokenHash = hashS3UploadToken(parsedToken.data); + const session = await findS3UploadSessionByTokenHash(tokenHash); + + if (!session) { + throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionNotFound); + } + + if (session.revokedAt) { + throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionRevoked); + } + + if (session.expiresAt.getTime() <= now.getTime()) { + throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionExpired); + } + + await markS3UploadSessionUsed(tokenHash); + + return S3ProxyUploadPayloadSchema.parse({ + bucketName: session.bucketName, + objectKey: session.objectKey, + maxSize: session.maxSize, + uploadConstraints: session.uploadConstraints, + metadata: session.metadata + }); +}; + +export const revokeS3UploadSessionToken = (token: string) => { + return revokeS3UploadSessionByTokenHash(hashS3UploadToken(token)); +}; diff --git a/packages/service/common/s3/accessLink/utils.ts b/packages/service/common/s3/accessLink/utils.ts new file mode 100644 index 000000000000..3554c1208e8f --- /dev/null +++ b/packages/service/common/s3/accessLink/utils.ts @@ -0,0 +1,169 @@ +import { addMinutes, differenceInMilliseconds } from 'date-fns'; +import { createHmac, timingSafeEqual } from 'node:crypto'; +import { serviceEnv } from '../../../env'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; +import { + S3_ACCESS_LINK_ROUTES, + S3_DOWNLOAD_ALIAS_ID_LENGTH, + S3_DOWNLOAD_ALIAS_SIGN_VERSION, + S3_DOWNLOAD_EXPIRE_BUCKET_MS, + S3_DOWNLOAD_EXPIRE_BUCKET_THRESHOLD_MS, + S3_DOWNLOAD_SIGNATURE_LENGTH, + S3_UPLOAD_TOKEN_LENGTH +} from './constants'; +import { + ParsedS3SignedDownloadAliasSchema, + S3SignedDownloadAliasValueSchema, + type ParsedS3SignedDownloadAlias +} from './type'; +import { S3AccessLinkErrCode, S3AccessLinkError } from './error'; + +const endpointUrl = `${serviceEnv.FILE_DOMAIN || serviceEnv.FE_DOMAIN || ''}${serviceEnv.NEXT_PUBLIC_BASE_URL}`; + +const hmacSha256Hex = (value: string) => + createHmac('sha256', serviceEnv.FILE_TOKEN_KEY).update(value).digest('hex'); + +const hmacSha256Base64Url = (value: string) => + createHmac('sha256', serviceEnv.FILE_TOKEN_KEY).update(value).digest('base64url'); + +const constantTimeEqual = (left: string, right: string) => { + const leftBuffer = Buffer.from(left); + const rightBuffer = Buffer.from(right); + + if (leftBuffer.length !== rightBuffer.length) return false; + + return timingSafeEqual(leftBuffer, rightBuffer); +}; + +const getDownloadExpireBucketMs = (ttlMs: number) => { + if (ttlMs <= S3_DOWNLOAD_EXPIRE_BUCKET_THRESHOLD_MS.short) { + return S3_DOWNLOAD_EXPIRE_BUCKET_MS.short; + } + + if (ttlMs <= S3_DOWNLOAD_EXPIRE_BUCKET_THRESHOLD_MS.medium) { + return S3_DOWNLOAD_EXPIRE_BUCKET_MS.medium; + } + + return S3_DOWNLOAD_EXPIRE_BUCKET_MS.long; +}; + +/** + * 把下载链接过期时间向上归一到时间桶。 + * + * 这样相同资源在短时间内反复签发时 URL 更稳定，同时不会让返回链接短于调用方请求的 + * `expiredTime`。如果调用方传入过去时间，则至少给 1 分钟有效期，保持与旧 JWT 签发的兜底语义一致。 + */ +export const resolveDownloadExpiresAt = (expiredTime: Date, now = new Date()) => { + const safeExpiredAt = new Date(Math.max(expiredTime.getTime(), addMinutes(now, 1).getTime())); + const ttlMs = Math.max(1, differenceInMilliseconds(safeExpiredAt, now)); + const bucketMs = getDownloadExpireBucketMs(ttlMs); + const expiresAtMs = Math.ceil(safeExpiredAt.getTime() / bucketMs) * bucketMs; + + return new Date(expiresAtMs); +}; + +export const encodeExpiresAtMinute = (date: Date) => { + const unixMinute = Math.ceil(date.getTime() / 60_000); + return unixMinute.toString(36); +}; + +export const decodeExpiresAtMinute = (value: string) => { + const unixMinute = Number.parseInt(value, 36); + + if (!Number.isFinite(unixMinute) || unixMinute <= 0) { + throw new S3AccessLinkError(S3AccessLinkErrCode.invalidSignedAlias); + } + + return new Date(unixMinute * 60_000); +}; + +export const generateS3AliasId = () => getNanoid(S3_DOWNLOAD_ALIAS_ID_LENGTH); + +export const generateS3UploadToken = () => getNanoid(S3_UPLOAD_TOKEN_LENGTH); + +export const hashS3UploadToken = (token: string) => hmacSha256Hex(token); + +export const buildS3DownloadAliasKey = ({ + bucketName, + objectKey, + filename, + responseContentType +}: { + bucketName: string; + objectKey: string; + filename?: string; + responseContentType?: string; +}) => { + const stablePayload = JSON.stringify({ + bucketName, + objectKey, + filename: filename ?? '', + responseContentType: responseContentType ?? '' + }); + + return hmacSha256Hex(stablePayload); +}; + +export const signS3DownloadAlias = ({ + aliasId, + expMinute36 +}: { + aliasId: string; + expMinute36: string; +}) => { + const signingInput = `s3-download:${S3_DOWNLOAD_ALIAS_SIGN_VERSION}:${aliasId}:${expMinute36}`; + return hmacSha256Base64Url(signingInput).slice(0, S3_DOWNLOAD_SIGNATURE_LENGTH); +}; + +export const parseSignedS3DownloadAlias = (value: string): ParsedS3SignedDownloadAlias => { + const parsedValue = S3SignedDownloadAliasValueSchema.safeParse(value); + + if (!parsedValue.success) { + throw new S3AccessLinkError(S3AccessLinkErrCode.invalidSignedAlias); + } + + const [aliasId, expMinute36, sig] = parsedValue.data.split('.'); + const parsed = ParsedS3SignedDownloadAliasSchema.safeParse({ aliasId, expMinute36, sig }); + + if (!parsed.success) { + throw new S3AccessLinkError(S3AccessLinkErrCode.invalidSignedAlias); + } + + return parsed.data; +}; + +/** + * 校验 signed alias 的格式、过期时间和 HMAC 签名。 + * + * Mongo alias 只负责资源映射；URL 的有效期由 `expMinute36` 和 `sig` 保证。 + */ +export const assertS3DownloadAliasSignature = (value: string, now = new Date()) => { + const parsed = parseSignedS3DownloadAlias(value); + const expiresAt = decodeExpiresAtMinute(parsed.expMinute36); + + if (expiresAt.getTime() <= now.getTime()) { + throw new S3AccessLinkError(S3AccessLinkErrCode.expiredSignedAlias); + } + + const expectedSig = signS3DownloadAlias({ + aliasId: parsed.aliasId, + expMinute36: parsed.expMinute36 + }); + + if (!constantTimeEqual(parsed.sig, expectedSig)) { + throw new S3AccessLinkError(S3AccessLinkErrCode.invalidSignedAliasSignature); + } + + return { + ...parsed, + expiresAt + }; +}; + +export const buildS3DownloadUrl = (signedAlias: string) => { + return `${endpointUrl}${S3_ACCESS_LINK_ROUTES.download}/${signedAlias}`; +}; + +export const buildS3UploadUrl = (token: string) => { + return `${endpointUrl}${S3_ACCESS_LINK_ROUTES.upload}/${token}`; +}; diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts index dd3724c6c9aa..158b1e3f171f 100644 --- a/packages/service/common/s3/buckets/base.ts +++ b/packages/service/common/s3/buckets/base.ts @@ -23,7 +23,11 @@ import { type UploadFileByBufferParams, UploadFileByBodySchema } from '../contra import type { createStorage } from '@fastgpt-sdk/storage'; import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools'; import { getContentDisposition } from '@fastgpt/global/common/file/tools'; -import { jwtSignS3DownloadToken, jwtSignS3UploadToken } from '../security/token'; +import { + createS3DownloadAccessUrl, + createS3UploadAccessUrl, + deleteS3DownloadAliasByObject +} from '../accessLink'; const logger = getLogger(LogCategories.INFRA.S3); @@ -143,6 +147,17 @@ export class S3BaseBucket { }); throw err; }); + + deleteS3DownloadAliasByObject({ + bucketName: this.bucketName, + objectKey + }).catch((err) => { + logger.warn('S3 download alias cleanup failed after object delete', { + key: objectKey, + bucketName: this.bucketName, + error: err + }); + }); } addDeleteJob(params: Omit[0], 'bucketName'>) { @@ -193,7 +208,7 @@ export class S3BaseBucket { }); return { - url: jwtSignS3UploadToken({ + url: await createS3UploadAccessUrl({ objectKey: params.rawKey, bucketName: this.bucketName, expiredTime: addMinutes(new Date(), Math.ceil(expiredSeconds / 60)), @@ -250,11 +265,12 @@ export class S3BaseBucket { return { bucket: this.bucketName, key, - url: jwtSignS3DownloadToken({ + url: await createS3DownloadAccessUrl({ objectKey: key, bucketName: this.bucketName, expiredTime: addMinutes(new Date(), Math.ceil(expires / 60)), - filename: path.basename(key) + filename: path.basename(key), + responseContentType }) }; } diff --git a/packages/service/common/s3/queue/delete.ts b/packages/service/common/s3/queue/delete.ts index b690d4443fdf..68ce977291ba 100644 --- a/packages/service/common/s3/queue/delete.ts +++ b/packages/service/common/s3/queue/delete.ts @@ -2,6 +2,7 @@ import { getQueue, getWorker, QueueNames } from '../../bullmq'; import { getLogger, LogCategories } from '../../logger'; import path from 'path'; import { batchRun } from '@fastgpt/global/common/system/utils'; +import { deleteS3DownloadAliasByObjects } from '../accessLink'; const logger = getLogger(LogCategories.INFRA.S3); @@ -64,6 +65,17 @@ export const executeS3DeleteJob = async ({ prefix, bucketName, key, keys }: S3MQ | undefined; assertNoFailedKeys(result?.keys, 'keys'); + deleteS3DownloadAliasByObjects({ + bucketName, + objectKeys: keys + }).catch((error) => { + logger.warn('S3 download alias cleanup failed after delete job', { + bucketName, + count: keys?.length, + error + }); + }); + await batchRun(keys, async (key) => { if (key.includes('-parsed/')) return; const fileParsedPrefix = `${path.dirname(key)}/${path.basename(key, path.extname(key))}-parsed`; diff --git a/packages/service/common/s3/security/token.ts b/packages/service/common/s3/security/token.ts index de105ca8c35b..d81c2027cdf1 100644 --- a/packages/service/common/s3/security/token.ts +++ b/packages/service/common/s3/security/token.ts @@ -1,16 +1,8 @@ import jwt from 'jsonwebtoken'; -import { differenceInSeconds } from 'date-fns'; import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode'; import type { UploadConstraints } from '../contracts/type'; -import path from 'path'; import { serviceEnv } from '../../../env'; -/* ==================== 路由与类型 ==================== */ -const FileApiPath = { - proxyDownload: '/api/system/file/download', - proxyUpload: '/api/system/file/upload' -} as const; - export type S3ObjectKeyTokenPayload = { objectKey: string; }; @@ -30,27 +22,7 @@ type S3UploadTokenPayload = { type: 'upload'; }; -type SignS3DownloadTokenParams = { - objectKey: string; - bucketName: string; - expiredTime: Date; - filename?: string; -}; - -type SignS3UploadTokenParams = { - objectKey: string; - bucketName: string; - expiredTime: Date; - maxSize: number; - uploadConstraints: UploadConstraints; - metadata?: Record; -}; - /* ==================== 通用工具函数 ==================== */ -const getExpiresIn = (expiredTime: Date) => { - return Math.max(1, differenceInSeconds(expiredTime, new Date())); -}; - const isRecord = (val: unknown): val is Record => !!val && typeof val === 'object' && !Array.isArray(val); @@ -60,10 +32,6 @@ const isStringArray = (val: unknown): val is string[] => const endpointUrl = `${serviceEnv.FILE_DOMAIN || serviceEnv.FE_DOMAIN || ''}${serviceEnv.NEXT_PUBLIC_BASE_URL}`; -const buildFileApiUrl = (apiPath: string, token: string, query = '') => { - return `${endpointUrl}${apiPath}/${token}${query}`; -}; - const parsePayload = (payload: unknown, checker: (value: unknown) => value is T): T => { if (!checker(payload)) { throw ERROR_ENUM.unAuthFile; @@ -71,12 +39,6 @@ const parsePayload = (payload: unknown, checker: (value: unknown) => value is return payload; }; -const signToken = (payload: T, expiredTime: Date) => { - return jwt.sign(payload, serviceEnv.FILE_TOKEN_KEY, { - expiresIn: getExpiresIn(expiredTime) - }); -}; - export const verifyToken = (token: string, checker: (value: unknown) => value is T) => { return new Promise((resolve, reject) => { jwt.verify(token, serviceEnv.FILE_TOKEN_KEY, (err, payload) => { @@ -127,55 +89,11 @@ const isS3UploadTokenPayload = (value: unknown): value is S3UploadTokenPayload = }; /* ==================== 代理下载 token ==================== */ -export function jwtSignS3DownloadToken({ - objectKey, - bucketName, - expiredTime, - filename -}: SignS3DownloadTokenParams) { - const token = signToken( - { - objectKey, - bucketName, - type: 'download' - } satisfies S3DownloadTokenPayload, - expiredTime - ); - - const finalFilename = filename || path.basename(objectKey) || ''; - const query = finalFilename ? `?filename=${encodeURIComponent(finalFilename)}` : ''; - - return buildFileApiUrl(FileApiPath.proxyDownload, token, query); -} - export function jwtVerifyS3DownloadToken(token: string) { return verifyToken(token, isS3DownloadTokenPayload); } /* ==================== 代理上传 token ==================== */ -export function jwtSignS3UploadToken({ - objectKey, - bucketName, - expiredTime, - maxSize, - uploadConstraints, - metadata -}: SignS3UploadTokenParams) { - const token = signToken( - { - objectKey, - bucketName, - maxSize, - uploadConstraints, - metadata, - type: 'upload' - } satisfies S3UploadTokenPayload, - expiredTime - ); - - return buildFileApiUrl(FileApiPath.proxyUpload, token); -} - export function jwtVerifyS3UploadToken(token: string) { return verifyToken(token, isS3UploadTokenPayload); } diff --git a/packages/service/core/dataset/data/controller.ts b/packages/service/core/dataset/data/controller.ts index 98a5219f86bf..6cad1e4922d4 100644 --- a/packages/service/core/dataset/data/controller.ts +++ b/packages/service/core/dataset/data/controller.ts @@ -3,11 +3,11 @@ import { addEndpointToImageUrl } from '../../../common/file/image/utils'; import type { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type'; import { addDays } from 'date-fns'; import { isS3ObjectKey } from '../../../common/s3/utils'; -import { jwtSignS3DownloadToken } from '../../../common/s3/security/token'; import { S3Buckets } from '../../../common/s3/config/constants'; import { matchDatasetDataMarkdownImages } from './utils'; +import { createS3DownloadAccessUrl } from '../../../common/s3/accessLink'; -export const formatDatasetDataValue = ({ +export const formatDatasetDataValue = async ({ q, a, imageId, @@ -17,11 +17,11 @@ export const formatDatasetDataValue = ({ a?: string; imageId?: string; imageDescMap?: Record; -}): { +}): Promise<{ q: string; a?: string; imagePreivewUrl?: string; -} => { +}> => { // Add image description to image markdown if (imageDescMap) { // Helper function to replace image markdown with description @@ -59,14 +59,19 @@ export const formatDatasetDataValue = ({ } if (!imageId) { + const [replacedQ, replacedA] = await Promise.all([ + replaceS3KeyToPreviewUrl(q, addDays(new Date(), 90)), + a ? replaceS3KeyToPreviewUrl(a, addDays(new Date(), 90)) : undefined + ]); + return { - q: replaceS3KeyToPreviewUrl(q, addDays(new Date(), 90)), - a: a ? replaceS3KeyToPreviewUrl(a, addDays(new Date(), 90)) : undefined + q: replacedQ, + a: replacedA }; } const imagePreivewUrl = isS3ObjectKey(imageId, 'dataset') - ? jwtSignS3DownloadToken({ + ? await createS3DownloadAccessUrl({ objectKey: imageId, bucketName: S3Buckets.private, expiredTime: addDays(new Date(), 90) @@ -81,15 +86,17 @@ export const formatDatasetDataValue = ({ }; export const getFormatDatasetCiteList = (list: DatasetDataSchemaType[]) => { - return list.map((item) => ({ - _id: item._id, - ...formatDatasetDataValue({ - q: item.q, - a: item.a, - imageId: item.imageId - }), - history: item.history, - updateTime: item.updateTime, - index: item.chunkIndex - })); + return Promise.all( + list.map(async (item) => ({ + _id: item._id, + ...(await formatDatasetDataValue({ + q: item.q, + a: item.a, + imageId: item.imageId + })), + history: item.history, + updateTime: item.updateTime, + index: item.chunkIndex + })) + ); }; diff --git a/packages/service/core/dataset/search/defaultRecall/embeddingRecall.ts b/packages/service/core/dataset/search/defaultRecall/embeddingRecall.ts index bb5892067d8e..18b5f208f3ab 100644 --- a/packages/service/core/dataset/search/defaultRecall/embeddingRecall.ts +++ b/packages/service/core/dataset/search/defaultRecall/embeddingRecall.ts @@ -245,36 +245,39 @@ export const embeddingRecall = async ({ image: [] }; - recallResults.forEach((recallResult, taskIndex) => { + for (const [taskIndex, recallResult] of recallResults.entries()) { const task = tasks[taskIndex]; const set = new Set(); - const list = recallResult.results - .map((item, index) => { - const collection = collectionMaps.get(String(item.collectionId)); - if (!collection) { - logger.warn('Dataset collection not found during recall', { - collectionId: item.collectionId, - dataId: item.id - }); - return; - } + const list = ( + await Promise.all( + recallResult.results.map((item, index) => { + const collection = collectionMaps.get(String(item.collectionId)); + if (!collection) { + logger.warn('Dataset collection not found during recall', { + collectionId: item.collectionId, + dataId: item.id + }); + return; + } - const data = dataMaps.get(String(item.id?.trim())); - if (!data) { - logger.warn('Dataset data not found during recall', { - dataId: item.id, - collectionId: item.collectionId - }); - return; - } + const data = dataMaps.get(String(item.id?.trim())); + if (!data) { + logger.warn('Dataset data not found during recall', { + dataId: item.id, + collectionId: item.collectionId + }); + return; + } - return buildSearchResultItem({ - data, - collection, - score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }] - }); - }) + return buildSearchResultItem({ + data, + collection, + score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }] + }); + }) + ) + ) .filter((item) => { if (!item) return false; if (set.has(item.id)) return false; @@ -289,7 +292,7 @@ export const embeddingRecall = async ({ }) as SearchDataResponseItemType[]; groupedRecallLists[task.source].push(list); - }); + } return { textEmbeddingRecallResults: concatRecallLists(groupedRecallLists.text, limit), diff --git a/packages/service/core/dataset/search/defaultRecall/fullTextRecall.ts b/packages/service/core/dataset/search/defaultRecall/fullTextRecall.ts index 06fccb1385ce..2d8e771a059f 100644 --- a/packages/service/core/dataset/search/defaultRecall/fullTextRecall.ts +++ b/packages/service/core/dataset/search/defaultRecall/fullTextRecall.ts @@ -159,41 +159,44 @@ export const fullTextRecall = async ({ imageCaption: [] }; - recallResults.forEach((recallResult, taskIndex) => { + for (const [taskIndex, recallResult] of recallResults.entries()) { const task = queryTasks[taskIndex]; - const list = recallResult - .map((item, index) => { - const collection = collectionMaps.get(String(item.collectionId)); - if (!collection) { - logger.warn('Dataset collection not found during full-text recall', { - collectionId: item.collectionId, - dataId: item.dataId - }); - return; - } + const list = ( + await Promise.all( + recallResult.map((item, index) => { + const collection = collectionMaps.get(String(item.collectionId)); + if (!collection) { + logger.warn('Dataset collection not found during full-text recall', { + collectionId: item.collectionId, + dataId: item.dataId + }); + return; + } - const data = dataMaps.get(String(item.dataId)); - if (!data) { - logger.warn('Dataset data not found during full-text recall', { - dataId: item.dataId, - collectionId: item.collectionId - }); - return; - } + const data = dataMaps.get(String(item.dataId)); + if (!data) { + logger.warn('Dataset data not found during full-text recall', { + dataId: item.dataId, + collectionId: item.collectionId + }); + return; + } - return buildSearchResultItem({ - data, - collection, - includeIndexes: true, - score: [ - { - type: SearchScoreTypeEnum.fullText, - value: item.score || 0, - index - } - ] - }); - }) + return buildSearchResultItem({ + data, + collection, + includeIndexes: true, + score: [ + { + type: SearchScoreTypeEnum.fullText, + value: item.score || 0, + index + } + ] + }); + }) + ) + ) .filter((item) => { if (!item) return false; return true; @@ -206,7 +209,7 @@ export const fullTextRecall = async ({ }) as SearchDataResponseItemType[]; groupedRecallLists[task.source].push(list); - }); + } return { textFullTextRecallResults: concatRecallLists(groupedRecallLists.text, limit), diff --git a/packages/service/core/dataset/search/defaultRecall/index.ts b/packages/service/core/dataset/search/defaultRecall/index.ts index 7f41e3150219..9176d75b8f2e 100644 --- a/packages/service/core/dataset/search/defaultRecall/index.ts +++ b/packages/service/core/dataset/search/defaultRecall/index.ts @@ -171,10 +171,12 @@ export async function searchDatasetData( const filterMaxTokensResult = await filterDatasetDataByMaxTokens(scoreFilter, maxTokens); // Step 8: 返回前把 q 中的内部图片 key 转为可预览 URL。 // 只在最终结果处理，避免中间召回和去重阶段混入带过期时间的动态 URL。 - const finalResult = filterMaxTokensResult.map((item) => { - item.q = replaceS3KeyToPreviewUrl(item.q, addDays(new Date(), 90)); - return item; - }); + const finalResult = await Promise.all( + filterMaxTokensResult.map(async (item) => { + item.q = await replaceS3KeyToPreviewUrl(item.q, addDays(new Date(), 90)); + return item; + }) + ); pushTrack.datasetSearch({ datasetIds, teamId }); diff --git a/packages/service/core/dataset/search/defaultRecall/result.ts b/packages/service/core/dataset/search/defaultRecall/result.ts index 40ce97aaa9bc..1a2880307050 100644 --- a/packages/service/core/dataset/search/defaultRecall/result.ts +++ b/packages/service/core/dataset/search/defaultRecall/result.ts @@ -23,22 +23,23 @@ export const buildSearchResultItem = ({ collection: DatasetCollectionSchemaType; score: SearchDataResponseItemType['score']; includeIndexes?: boolean; -}): SearchDataResponseItemType => ({ - id: String(data._id), - updateTime: data.updateTime, - ...formatDatasetDataValue({ +}): Promise => + formatDatasetDataValue({ q: data.q, a: data.a, imageId: data.imageId, imageDescMap: data.imageDescMap - }), - chunkIndex: data.chunkIndex, - ...(includeIndexes ? { indexes: data.indexes } : {}), - datasetId: String(data.datasetId), - collectionId: String(data.collectionId), - ...getCollectionSourceData(collection), - score -}); + }).then((formattedValue) => ({ + id: String(data._id), + updateTime: data.updateTime, + ...formattedValue, + chunkIndex: data.chunkIndex, + ...(includeIndexes ? { indexes: data.indexes } : {}), + datasetId: String(data.datasetId), + collectionId: String(data.collectionId), + ...getCollectionSourceData(collection), + score + })); export const concatRecallLists = (lists: SearchDataResponseItemType[][], limit: number) => { return datasetSearchResultConcat(lists.map((list) => ({ weight: 1, list }))).slice(0, limit); diff --git a/packages/service/core/dataset/utils.ts b/packages/service/core/dataset/utils.ts index 483a4c05f3eb..93e55f82f29c 100644 --- a/packages/service/core/dataset/utils.ts +++ b/packages/service/core/dataset/utils.ts @@ -2,11 +2,11 @@ import { authDatasetByTmbId } from '../../support/permission/dataset/auth'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { S3Sources } from '../../common/s3/contracts/type'; import { isS3ObjectKey } from '../../common/s3/utils'; -import { jwtSignS3DownloadToken } from '../../common/s3/security/token'; import { getLogger, LogCategories } from '../../common/logger'; import { S3Buckets } from '../../common/s3/config/constants'; import { getVlmModelList, isImageEmbeddingModel } from '../ai/model'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { createS3DownloadAccessUrl } from '../../common/s3/accessLink'; const logger = getLogger(LogCategories.MODULE.DATASET.FILE); @@ -39,7 +39,7 @@ export const filterDatasetsByTmbId = async ({ }; /** - * 替换数据集引用 markdown 文本中的图片链接格式的 S3 对象键为 JWT 签名后的 URL + * 替换数据集引用 markdown 文本中的图片链接格式的 S3 对象键为短访问 URL。 * * @param documentQuoteText 数据集引用文本 * @param expiredTime 过期时间 @@ -51,10 +51,10 @@ export const filterDatasetsByTmbId = async ({ * const datasetQuoteText = '![image.png](dataset/68fee42e1d416bb5ddc85b19/6901c3071ba2bea567e8d8db/aZos7D-214afce5-4d42-4356-9e05-8164d51c59ae.png)'; * const replacedText = await replaceS3KeyToPreviewUrl(datasetQuoteText, addDays(new Date(), 90)) * console.log(replacedText) - * // '![image.png](http://localhost:3000/api/system/file/download/xxx?filename=image.png)' + * // '![image.png](http://localhost:3000/api/system/file/d/alias.exp.sig)' * ``` */ -export function replaceS3KeyToPreviewUrl(documentQuoteText: string, expiredTime: Date) { +export async function replaceS3KeyToPreviewUrl(documentQuoteText: string, expiredTime: Date) { if (!documentQuoteText || typeof documentQuoteText !== 'string') return documentQuoteText as string; @@ -72,7 +72,7 @@ export function replaceS3KeyToPreviewUrl(documentQuoteText: string, expiredTime: const allowedKeysGuard = allowedKeys.some((key) => isS3ObjectKey(objectKey, key)); if (allowedKeysGuard) { - const url = jwtSignS3DownloadToken({ + const url = await createS3DownloadAccessUrl({ objectKey, bucketName: S3Buckets.private, expiredTime diff --git a/packages/service/core/workflow/utils/file.ts b/packages/service/core/workflow/utils/file.ts index 9412b8fe71f2..0a83612ad4e2 100644 --- a/packages/service/core/workflow/utils/file.ts +++ b/packages/service/core/workflow/utils/file.ts @@ -209,7 +209,7 @@ export const getFileContentByUrl = async ({ usageId }); - const replacedText = replaceS3KeyToPreviewUrl(rawText, addDays(new Date(), 90)); + const replacedText = await replaceS3KeyToPreviewUrl(rawText, addDays(new Date(), 90)); // Add to buffer getS3RawTextSource().addRawTextBuffer({ diff --git a/packages/service/test/common/s3/accessLink.test.ts b/packages/service/test/common/s3/accessLink.test.ts new file mode 100644 index 000000000000..c0fde25592e5 --- /dev/null +++ b/packages/service/test/common/s3/accessLink.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const strongFileTokenKey = '1234567890abcdef1234567890abcdef'; +const originalEnv = { + FILE_TOKEN_KEY: process.env.FILE_TOKEN_KEY, + FILE_DOMAIN: process.env.FILE_DOMAIN, + FE_DOMAIN: process.env.FE_DOMAIN, + NEXT_PUBLIC_BASE_URL: process.env.NEXT_PUBLIC_BASE_URL +}; + +const loadAccessLinkModules = async () => { + vi.resetModules(); + vi.stubEnv('FILE_TOKEN_KEY', strongFileTokenKey); + vi.stubEnv('FILE_DOMAIN', 'https://files.example.com/'); + vi.stubEnv('FE_DOMAIN', undefined); + vi.stubEnv('NEXT_PUBLIC_BASE_URL', '/fastgpt'); + + const [accessLink, downloadAliasSchema, uploadSessionSchema] = await Promise.all([ + import('@fastgpt/service/common/s3/accessLink'), + import('@fastgpt/service/common/s3/accessLink/downloadAlias/schema'), + import('@fastgpt/service/common/s3/accessLink/uploadSession/schema') + ]); + + return { + ...accessLink, + MongoS3DownloadAlias: downloadAliasSchema.MongoS3DownloadAlias, + MongoS3UploadSession: uploadSessionSchema.MongoS3UploadSession + }; +}; + +const getFutureDate = (minutes: number) => new Date(Date.now() + minutes * 60 * 1000); + +const extractLastPathSegment = (url: string) => url.split('/').pop() || ''; + +describe('s3 access link', () => { + beforeEach(() => { + vi.resetModules(); + }); + + afterEach(() => { + vi.stubEnv('FILE_TOKEN_KEY', originalEnv.FILE_TOKEN_KEY); + vi.stubEnv('FILE_DOMAIN', originalEnv.FILE_DOMAIN); + vi.stubEnv('FE_DOMAIN', originalEnv.FE_DOMAIN); + vi.stubEnv('NEXT_PUBLIC_BASE_URL', originalEnv.NEXT_PUBLIC_BASE_URL); + vi.restoreAllMocks(); + }); + + it('creates stable short download URLs backed by one alias document', async () => { + const { + createS3DownloadAccessUrl, + verifyS3DownloadAccess, + revokeS3DownloadAlias, + MongoS3DownloadAlias + } = await loadAccessLinkModules(); + const params = { + bucketName: 'fastgpt-private', + objectKey: 'dataset/team-1/file.png', + expiredTime: getFutureDate(30), + filename: 'file.png', + responseContentType: 'image/png' + }; + + const firstUrl = await createS3DownloadAccessUrl(params); + const secondUrl = await createS3DownloadAccessUrl(params); + + expect(firstUrl).toBe(secondUrl); + expect(firstUrl).toMatch( + /^https:\/\/files\.example\.com\/fastgpt\/api\/system\/file\/d\/[A-Za-z0-9_-]{16}\.[0-9a-z]+\.[A-Za-z0-9_-]{22}$/ + ); + expect(firstUrl).not.toContain(params.objectKey); + + const aliases = await MongoS3DownloadAlias.find({}).lean(); + expect(aliases).toHaveLength(1); + expect(aliases[0]?.bucketName).toBe(params.bucketName); + expect(aliases[0]?.objectKey).toBe(params.objectKey); + expect(aliases[0]?.purgeAt.getTime()).toBeGreaterThan(params.expiredTime.getTime()); + + await expect(verifyS3DownloadAccess(extractLastPathSegment(firstUrl))).resolves.toMatchObject({ + bucketName: params.bucketName, + objectKey: params.objectKey, + filename: params.filename, + responseContentType: params.responseContentType + }); + + const aliasId = extractLastPathSegment(firstUrl).split('.')[0] || ''; + await revokeS3DownloadAlias(aliasId); + + await expect(verifyS3DownloadAccess(extractLastPathSegment(firstUrl))).rejects.toThrow( + 'DownloadAliasRevoked' + ); + }); + + it('extends download alias purgeAt without creating another alias document', async () => { + const { createS3DownloadAccessUrl, MongoS3DownloadAlias } = await loadAccessLinkModules(); + const params = { + bucketName: 'fastgpt-private', + objectKey: 'dataset/team-1/long-lived.png', + filename: 'long-lived.png' + }; + + await createS3DownloadAccessUrl({ + ...params, + expiredTime: getFutureDate(10) + }); + const firstAlias = await MongoS3DownloadAlias.findOne({ objectKey: params.objectKey }).lean(); + + await createS3DownloadAccessUrl({ + ...params, + expiredTime: getFutureDate(60 * 48) + }); + const aliases = await MongoS3DownloadAlias.find({ objectKey: params.objectKey }).lean(); + + expect(aliases).toHaveLength(1); + expect(aliases[0]?.purgeAt.getTime()).toBeGreaterThan(firstAlias?.purgeAt.getTime() || 0); + }); + + it('rejects expired or tampered signed aliases before alias lookup', async () => { + const { assertS3DownloadAliasSignature, encodeExpiresAtMinute, signS3DownloadAlias } = + await loadAccessLinkModules(); + const aliasId = 'R7mQG0Yh2kVxP9Za'; + const expMinute36 = encodeExpiresAtMinute(getFutureDate(10)); + const sig = signS3DownloadAlias({ aliasId, expMinute36 }); + const tamperedExpMinute36 = (Number.parseInt(expMinute36, 36) + 60).toString(36); + const expiredExpMinute36 = encodeExpiresAtMinute(getFutureDate(-10)); + const expiredSig = signS3DownloadAlias({ + aliasId, + expMinute36: expiredExpMinute36 + }); + + expect(() => + assertS3DownloadAliasSignature(`${aliasId}.${tamperedExpMinute36}.${sig}`) + ).toThrow('InvalidSignedAliasSignature'); + expect(() => + assertS3DownloadAliasSignature(`${aliasId}.${expiredExpMinute36}.${expiredSig}`) + ).toThrow('ExpiredSignedAlias'); + }); + + it('stores upload session payload behind a short hashed token', async () => { + const { + createS3UploadAccessUrl, + verifyS3UploadSessionToken, + parseSignedS3DownloadAlias, + MongoS3UploadSession + } = await loadAccessLinkModules(); + + const url = await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/file.txt', + expiredTime: getFutureDate(10), + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'text/plain', + allowedExtensions: ['.txt'] + }, + metadata: { + originFilename: 'file.txt' + } + }); + const token = extractLastPathSegment(url); + + expect(url).toMatch( + /^https:\/\/files\.example\.com\/fastgpt\/api\/system\/file\/u\/[A-Za-z0-9_-]{22}$/ + ); + expect(() => parseSignedS3DownloadAlias(token)).toThrow('InvalidSignedAlias'); + + const sessions = await MongoS3UploadSession.find({}).lean(); + expect(sessions).toHaveLength(1); + expect(sessions[0]?.tokenHash).toMatch(/^[a-f0-9]{64}$/); + expect(sessions[0]?.tokenHash).not.toBe(token); + + await expect(verifyS3UploadSessionToken(token)).resolves.toMatchObject({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/file.txt', + maxSize: 1024 + }); + + const usedSession = await MongoS3UploadSession.findOne({ + tokenHash: sessions[0]?.tokenHash + }).lean(); + expect(usedSession?.usedAt).toBeInstanceOf(Date); + }); + + it('rejects expired and revoked upload sessions', async () => { + const { createS3UploadAccessUrl, verifyS3UploadSessionToken, revokeS3UploadSessionToken } = + await loadAccessLinkModules(); + const expiredToken = extractLastPathSegment( + await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/expired.txt', + expiredTime: getFutureDate(-10), + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'text/plain' + } + }) + ); + const revokedToken = extractLastPathSegment( + await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/revoked.txt', + expiredTime: getFutureDate(10), + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'text/plain' + } + }) + ); + + await revokeS3UploadSessionToken(revokedToken); + + await expect(verifyS3UploadSessionToken(expiredToken)).rejects.toThrow('UploadSessionExpired'); + await expect(verifyS3UploadSessionToken(revokedToken)).rejects.toThrow('UploadSessionRevoked'); + }); +}); diff --git a/packages/service/test/common/s3/token.test.ts b/packages/service/test/common/s3/token.test.ts index 7b14f828eaef..a2a8575d21ff 100644 --- a/packages/service/test/common/s3/token.test.ts +++ b/packages/service/test/common/s3/token.test.ts @@ -1,8 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode'; +import jwt from 'jsonwebtoken'; const strongFileTokenKey = '1234567890abcdef1234567890abcdef'; -const getExpiredTime = () => new Date(Date.now() + 5 * 60 * 1000); const originalEnv = { FILE_TOKEN_KEY: process.env.FILE_TOKEN_KEY, FILE_DOMAIN: process.env.FILE_DOMAIN, @@ -10,10 +10,6 @@ const originalEnv = { NEXT_PUBLIC_BASE_URL: process.env.NEXT_PUBLIC_BASE_URL }; -const extractTokenFromUrl = (url: string) => { - return url.split('/').pop()?.split('?')[0] || ''; -}; - const loadTokenModule = async () => { vi.resetModules(); vi.stubEnv('FILE_TOKEN_KEY', strongFileTokenKey); @@ -35,50 +31,43 @@ describe('s3 token validation', () => { }); it('rejects upload tokens when verifying download tokens', async () => { - const { jwtSignS3UploadToken, jwtVerifyS3DownloadToken } = await loadTokenModule(); - const token = extractTokenFromUrl( - jwtSignS3UploadToken({ + const { jwtVerifyS3DownloadToken } = await loadTokenModule(); + const token = jwt.sign( + { objectKey: 'chat/appId/userId/chatId/file.txt', bucketName: 'fastgpt-private', - expiredTime: getExpiredTime(), maxSize: 1024, uploadConstraints: { defaultContentType: 'text/plain' - } - }) + }, + type: 'upload' + }, + strongFileTokenKey, + { expiresIn: 300 } ); await expect(jwtVerifyS3DownloadToken(token)).rejects.toBe(ERROR_ENUM.unAuthFile); }); it('rejects download tokens when verifying upload tokens', async () => { - const { jwtSignS3DownloadToken, jwtVerifyS3UploadToken } = await loadTokenModule(); - const token = extractTokenFromUrl( - jwtSignS3DownloadToken({ + const { jwtVerifyS3UploadToken } = await loadTokenModule(); + const token = jwt.sign( + { objectKey: 'dataset/datasetId/file.txt', bucketName: 'fastgpt-private', - expiredTime: getExpiredTime(), - filename: 'file.txt' - }) + type: 'download' + }, + strongFileTokenKey, + { expiresIn: 300 } ); await expect(jwtVerifyS3UploadToken(token)).rejects.toBe(ERROR_ENUM.unAuthFile); }); - it('normalizes endpoint slashes when signing proxy download URLs', async () => { - vi.stubEnv('FILE_DOMAIN', 'https://files.example.com/'); - vi.stubEnv('FE_DOMAIN', undefined); - vi.stubEnv('NEXT_PUBLIC_BASE_URL', '/fastgpt'); + it('does not expose legacy JWT signing helpers', async () => { + const tokenModule = await loadTokenModule(); - const { jwtSignS3DownloadToken } = await loadTokenModule(); - const url = jwtSignS3DownloadToken({ - objectKey: 'chat/appId/userId/chatId/file.txt', - bucketName: 'fastgpt-private', - expiredTime: getExpiredTime() - }); - - expect(url).toMatch( - /^https:\/\/files\.example\.com\/fastgpt\/api\/system\/file\/download\/[^/?#]+\?filename=file\.txt$/ - ); + expect('jwtSignS3DownloadToken' in tokenModule).toBe(false); + expect('jwtSignS3UploadToken' in tokenModule).toBe(false); }); }); diff --git a/packages/service/test/core/dataset/data/controller.test.ts b/packages/service/test/core/dataset/data/controller.test.ts index 424ba05543da..74768774866c 100644 --- a/packages/service/test/core/dataset/data/controller.test.ts +++ b/packages/service/test/core/dataset/data/controller.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it } from 'vitest'; import { formatDatasetDataValue } from '@fastgpt/service/core/dataset/data/controller'; describe('formatDatasetDataValue', () => { - it('should append image descriptions to markdown image alt text in question and answer', () => { - const result = formatDatasetDataValue({ + it('should append image descriptions to markdown image alt text in question and answer', async () => { + const result = await formatDatasetDataValue({ q: 'Question ![cat]( https://example.com/cat.png ) and ![bird](https://example.com/bird.png)', a: 'Answer ![](https://example.com/dog.png)', imageDescMap: { diff --git a/packages/service/test/core/dataset/utils.test.ts b/packages/service/test/core/dataset/utils.test.ts index 03a62521582a..62efab50cd0a 100644 --- a/packages/service/test/core/dataset/utils.test.ts +++ b/packages/service/test/core/dataset/utils.test.ts @@ -22,11 +22,10 @@ vi.mock('@fastgpt/service/common/s3/utils', () => ({ }) })); -vi.mock('@fastgpt/service/common/s3/security/token', () => ({ - jwtSignS3DownloadToken: vi.fn( - ({ objectKey }: { objectKey: string }) => - `https://example.com/api/system/file/download/mock-jwt-token-${objectKey}` - ) +vi.mock('@fastgpt/service/common/s3/accessLink', () => ({ + createS3DownloadAccessUrl: vi.fn(async ({ objectKey }: { objectKey: string }) => { + return `https://example.com/api/system/file/d/mock-short-link-${objectKey}`; + }) })); vi.mock('@fastgpt/service/common/s3/contracts/type', () => ({ @@ -47,84 +46,84 @@ describe('replaceS3KeyToPreviewUrl', () => { }); describe('边界情况处理', () => { - it('空字符串应返回空字符串', () => { - const result = replaceS3KeyToPreviewUrl('', expiredTime); + it('空字符串应返回空字符串', async () => { + const result = await replaceS3KeyToPreviewUrl('', expiredTime); expect(result).toBe(''); }); - it('null 应返回 null', () => { - const result = replaceS3KeyToPreviewUrl(null as unknown as string, expiredTime); + it('null 应返回 null', async () => { + const result = await replaceS3KeyToPreviewUrl(null as unknown as string, expiredTime); expect(result).toBe(null); }); - it('undefined 应返回 undefined', () => { - const result = replaceS3KeyToPreviewUrl(undefined as unknown as string, expiredTime); + it('undefined 应返回 undefined', async () => { + const result = await replaceS3KeyToPreviewUrl(undefined as unknown as string, expiredTime); expect(result).toBe(undefined); }); - it('非字符串类型应原样返回', () => { - const result = replaceS3KeyToPreviewUrl(123 as unknown as string, expiredTime); + it('非字符串类型应原样返回', async () => { + const result = await replaceS3KeyToPreviewUrl(123 as unknown as string, expiredTime); expect(result).toBe(123); }); }); // 测试不包含 S3 链接的普通文本 describe('普通文本处理', () => { - it('纯文本不做任何替换', () => { + it('纯文本不做任何替换', async () => { const text = '这是一段普通文本，不包含任何图片链接'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toBe(text); }); - it('普通 HTTP 链接不做替换', () => { + it('普通 HTTP 链接不做替换', async () => { const text = '![image](https://example.com/image.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toBe(text); }); - it('普通 markdown 链接不做替换', () => { + it('普通 markdown 链接不做替换', async () => { const text = '[链接文本](https://example.com/page)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toBe(text); }); }); // 测试 dataset 前缀的 S3 链接替换 describe('dataset S3 链接替换', () => { - it('应替换 dataset 图片链接', () => { + it('应替换 dataset 图片链接', async () => { const text = '![image.png](dataset/68fee42e1d416bb5ddc85b19/6901c3071ba2bea567e8d8db/aZos7D-214afce5-4d42-4356-9e05-8164d51c59ae.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); expect(result).toContain('dataset/68fee42e1d416bb5ddc85b19'); expect(result).toMatch(/!\[image\.png\]\(https:\/\/example\.com/); }); - it('应替换 dataset 普通链接（非图片）', () => { + it('应替换 dataset 普通链接（非图片）', async () => { const text = '[文档](dataset/68fee42e1d416bb5ddc85b19/6901c3071ba2bea567e8d8db/document.pdf)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); expect(result).toMatch(/\[文档\]\(https:\/\/example\.com/); }); }); // 测试 chat 前缀的 S3 链接替换 describe('chat S3 链接替换', () => { - it('应替换 chat 图片链接', () => { + it('应替换 chat 图片链接', async () => { const text = '![screenshot.png](chat/691ae29d404d0468717dd747/68ad85a7463006c96379a07/jXfXy8yfGAFs9WJpcWRbAhV2/parsed/9a0f4fed-4edf-4613-a8d6-533af5ae51dc.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); expect(result).toContain('chat/691ae29d404d0468717dd747'); }); }); // 测试多个链接替换 describe('多个链接替换', () => { - it('应正确替换多个 S3 链接', () => { + it('应正确替换多个 S3 链接', async () => { const text = `这是一段包含多个图片的文本： ![图片1](dataset/team1/collection1/image1.png) 一些中间文字 @@ -132,11 +131,11 @@ describe('replaceS3KeyToPreviewUrl', () => { 更多文字 ![外部图片](https://external.com/image3.png)`; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); // dataset 和 chat 链接应被替换 - expect(result).toContain('mock-jwt-token-dataset/team1/collection1/image1.png'); - expect(result).toContain('mock-jwt-token-chat/app1/user1/chat1/image2.jpg'); + expect(result).toContain('mock-short-link-dataset/team1/collection1/image1.png'); + expect(result).toContain('mock-short-link-chat/app1/user1/chat1/image2.jpg'); // 外部链接不应被替换 expect(result).toContain('https://external.com/image3.png'); }); @@ -144,17 +143,17 @@ describe('replaceS3KeyToPreviewUrl', () => { // 测试不支持的 S3 前缀 describe('不支持的 S3 前缀', () => { - it('avatar 前缀不应被替换（只支持 dataset 和 chat）', () => { + it('avatar 前缀不应被替换（只支持 dataset 和 chat）', async () => { const text = '![头像](avatar/team1/user-avatar.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); // avatar 的 isS3ObjectKey 返回 false（因为只检查 dataset 和 chat） expect(result).toBe(text); }); - it('temp 前缀应被替换', () => { + it('temp 前缀应被替换', async () => { const text = '![临时文件](temp/team1/temp-file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); expect(result).toContain('temp/team1/temp-file.png'); }); }); @@ -162,280 +161,280 @@ describe('replaceS3KeyToPreviewUrl', () => { // 测试特殊字符处理 describe('特殊字符处理', () => { // 中文字符 - it('文件名包含中文应正常处理', () => { + it('文件名包含中文应正常处理', async () => { const text = '![中文图片名.png](dataset/team1/collection1/中文文件名.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); }); - it('alt 文本为空应正常处理', () => { + it('alt 文本为空应正常处理', async () => { const text = '![](dataset/team1/collection1/no-alt.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toMatch(/!\[\]\(https:\/\/example\.com/); }); // 日韩文字符 - it('文件名包含日文应正常处理', () => { + it('文件名包含日文应正常处理', async () => { const text = '![日本語ファイル](dataset/team1/日本語テスト.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/日本語テスト.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/日本語テスト.png'); }); - it('文件名包含韩文应正常处理', () => { + it('文件名包含韩文应正常处理', async () => { const text = '![한국어](dataset/team1/한국어파일.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/한국어파일.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/한국어파일.png'); }); // Emoji 表情符号 - it('文件名包含 emoji 应正常处理', () => { + it('文件名包含 emoji 应正常处理', async () => { const text = '![🎉 celebration](dataset/team1/🎉emoji🚀test.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/🎉emoji🚀test.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/🎉emoji🚀test.png'); }); - it('alt 文本包含多个 emoji 应正常处理', () => { + it('alt 文本包含多个 emoji 应正常处理', async () => { const text = '![🔥💯🎯](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toMatch(/!\[🔥💯🎯\]\(https:\/\/example\.com/); }); // 特殊符号 - it('文件名包含下划线和连字符应正常处理', () => { + it('文件名包含下划线和连字符应正常处理', async () => { const text = '![image](dataset/team1/my_file-name_v2.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/my_file-name_v2.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/my_file-name_v2.png'); }); - it('文件名包含 @ 符号应正常处理', () => { + it('文件名包含 @ 符号应正常处理', async () => { const text = '![email](dataset/team1/user@example.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/user@example.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/user@example.png'); }); - it('文件名包含 # 符号应正常处理', () => { + it('文件名包含 # 符号应正常处理', async () => { const text = '![hash](dataset/team1/file#1.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/file#1.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/file#1.png'); }); - it('文件名包含 $ 符号应正常处理', () => { + it('文件名包含 $ 符号应正常处理', async () => { const text = '![dollar](dataset/team1/price$100.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/price$100.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/price$100.png'); }); - it('文件名包含 % 符号应正常处理', () => { + it('文件名包含 % 符号应正常处理', async () => { const text = '![percent](dataset/team1/50%off.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/50%off.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/50%off.png'); }); - it('文件名包含 + 符号应正常处理', () => { + it('文件名包含 + 符号应正常处理', async () => { const text = '![plus](dataset/team1/a+b.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/a+b.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/a+b.png'); }); - it('文件名包含 = 符号应正常处理', () => { + it('文件名包含 = 符号应正常处理', async () => { const text = '![equals](dataset/team1/x=1.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/x=1.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/x=1.png'); }); // 多个点号 - it('文件名包含多个点号应正常处理', () => { + it('文件名包含多个点号应正常处理', async () => { const text = '![dots](dataset/team1/file.name.v1.2.3.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/file.name.v1.2.3.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/file.name.v1.2.3.png'); }); // 空格相关 - it('alt 文本包含空格应正常处理', () => { + it('alt 文本包含空格应正常处理', async () => { const text = '![image with spaces](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toMatch(/!\[image with spaces\]\(https:\/\/example\.com/); }); - it('文件名包含 URL 编码的空格 %20 应正常处理', () => { + it('文件名包含 URL 编码的空格 %20 应正常处理', async () => { const text = '![encoded](dataset/team1/file%20name.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/file%20name.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/file%20name.png'); }); // 括号类字符 - it('alt 文本包含转义方括号不匹配正则，不做替换', () => { + it('alt 文本包含转义方括号不匹配正则，不做替换', async () => { // 由于 markdown 正则 [^\]]* 不匹配包含 ] 的 alt 文本，这种情况不会被替换 const text = '![image \\[1\\]](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); // 预期不做替换 expect(result).toBe(text); }); - it('alt 文本包含圆括号应正常处理', () => { + it('alt 文本包含圆括号应正常处理', async () => { const text = '![image (1)](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); }); - it('文件名包含花括号应正常处理', () => { + it('文件名包含花括号应正常处理', async () => { const text = '![braces](dataset/team1/file{1}.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/file{1}.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/file{1}.png'); }); - it('文件名包含方括号应正常处理', () => { + it('文件名包含方括号应正常处理', async () => { const text = '![braces](dataset/team1/file[1].png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/file[1].png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/file[1].png'); }); // 引号 - it('alt 文本包含单引号应正常处理', () => { + it('alt 文本包含单引号应正常处理', async () => { const text = "![it's a test](dataset/team1/file.png)"; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toMatch(/!\[it's a test\]\(https:\/\/example\.com/); }); - it('alt 文本包含双引号应正常处理', () => { + it('alt 文本包含双引号应正常处理', async () => { const text = '![say "hello"](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); }); // 反斜杠 - it('alt 文本包含反斜杠应正常处理', () => { + it('alt 文本包含反斜杠应正常处理', async () => { const text = '![path\\to\\file](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); }); // 特殊 markdown 字符 - it('alt 文本包含星号应正常处理', () => { + it('alt 文本包含星号应正常处理', async () => { const text = '![*important*](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toMatch(/!\[\*important\*\]\(https:\/\/example\.com/); }); - it('alt 文本包含下划线强调应正常处理', () => { + it('alt 文本包含下划线强调应正常处理', async () => { const text = '![_emphasis_](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toMatch(/!\[_emphasis_\]\(https:\/\/example\.com/); }); - it('alt 文本包含反引号应正常处理', () => { + it('alt 文本包含反引号应正常处理', async () => { const text = '![`code`](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toMatch(/!\[`code`\]\(https:\/\/example\.com/); }); // 数字和字母混合 - it('文件名是纯 UUID 格式应正常处理', () => { + it('文件名是纯 UUID 格式应正常处理', async () => { const text = '![uuid](dataset/team1/550e8400-e29b-41d4-a716-446655440000.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toContain( - 'mock-jwt-token-dataset/team1/550e8400-e29b-41d4-a716-446655440000.png' + 'mock-short-link-dataset/team1/550e8400-e29b-41d4-a716-446655440000.png' ); }); - it('文件名是纯数字应正常处理', () => { + it('文件名是纯数字应正常处理', async () => { const text = '![numbers](dataset/team1/123456789.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/123456789.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/123456789.png'); }); // 超长文件名 - it('超长文件名应正常处理', () => { + it('超长文件名应正常处理', async () => { const longName = 'a'.repeat(200); const text = `![long](dataset/team1/${longName}.png)`; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain(`mock-jwt-token-dataset/team1/${longName}.png`); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain(`mock-short-link-dataset/team1/${longName}.png`); }); // 阿拉伯文和希伯来文（RTL 文字） - it('文件名包含阿拉伯文应正常处理', () => { + it('文件名包含阿拉伯文应正常处理', async () => { const text = '![عربي](dataset/team1/ملف.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/ملف.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/ملف.png'); }); // 俄文 - it('文件名包含俄文应正常处理', () => { + it('文件名包含俄文应正常处理', async () => { const text = '![русский](dataset/team1/файл.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/файл.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/файл.png'); }); // 泰文 - it('文件名包含泰文应正常处理', () => { + it('文件名包含泰文应正常处理', async () => { const text = '![ไทย](dataset/team1/ไฟล์.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/ไฟล์.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/ไฟล์.png'); }); // 特殊扩展名 - it('无扩展名的文件应正常处理', () => { + it('无扩展名的文件应正常处理', async () => { const text = '![noext](dataset/team1/README)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/README'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/README'); }); - it('双扩展名的文件应正常处理', () => { + it('双扩展名的文件应正常处理', async () => { const text = '![tarball](dataset/team1/archive.tar.gz)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/archive.tar.gz'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/archive.tar.gz'); }); // 管道符和其他 shell 特殊字符 - it('文件名包含管道符应正常处理', () => { + it('文件名包含管道符应正常处理', async () => { const text = '![pipe](dataset/team1/a|b.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/a|b.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/a|b.png'); }); - it('文件名包含波浪号应正常处理', () => { + it('文件名包含波浪号应正常处理', async () => { const text = '![tilde](dataset/team1/~user.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/~user.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/~user.png'); }); - it('文件名包含 & 符号应正常处理', () => { + it('文件名包含 & 符号应正常处理', async () => { const text = '![ampersand](dataset/team1/a&b.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/a&b.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/a&b.png'); }); // 换行符 - it('alt 文本不包含换行符时应正常处理', () => { + it('alt 文本不包含换行符时应正常处理', async () => { const text = '![single line](dataset/team1/file.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); }); // 特殊组合 - it('文件名包含多种特殊字符组合应正常处理', () => { + it('文件名包含多种特殊字符组合应正常处理', async () => { const text = '![complex](dataset/team1/file_v1.2-beta@test#1$100%off.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/file_v1.2-beta@test#1$100%off.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/file_v1.2-beta@test#1$100%off.png'); }); - it('中英文混合 alt 和文件名应正常处理', () => { + it('中英文混合 alt 和文件名应正常处理', async () => { const text = '![测试image图片](dataset/team1/test测试file文件.png)'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('mock-jwt-token-dataset/team1/test测试file文件.png'); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); + expect(result).toContain('mock-short-link-dataset/team1/test测试file文件.png'); }); }); // 测试链接格式边界情况 describe('链接格式边界情况', () => { - it('链接中有空格应正常处理', () => { + it('链接中有空格应正常处理', async () => { const text = '![image]( dataset/team1/collection1/image.png )'; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); - expect(result).toContain('https://example.com/api/system/file/download/mock-jwt-token-'); + expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); }); - it('混合文本和链接应只替换 S3 链接', () => { + it('混合文本和链接应只替换 S3 链接', async () => { const text = `# 标题 普通段落文字 ![S3图片](dataset/team1/file.png) 后续文字 @@ -446,10 +445,10 @@ describe('replaceS3KeyToPreviewUrl', () => { 代码块 \`\`\``; - const result = replaceS3KeyToPreviewUrl(text, expiredTime); + const result = await replaceS3KeyToPreviewUrl(text, expiredTime); expect(result).toContain( - 'https://example.com/api/system/file/download/mock-jwt-token-dataset/team1/file.png' + 'https://example.com/api/system/file/d/mock-short-link-dataset/team1/file.png' ); expect(result).toContain('https://google.com'); expect(result).toContain('# 标题'); @@ -458,7 +457,7 @@ describe('replaceS3KeyToPreviewUrl', () => { }); describe('matchDatasetDataMarkdownImageUrls', () => { - it('应提取统一的 markdown 图片节点结构', () => { + it('应提取统一的 markdown 图片节点结构', async () => { const result = matchDatasetDataMarkdownImages( '文字 ![猫]( dataset/team/cat.png ) 和 ![dog](https://example.com/dog.png)' ); @@ -479,7 +478,7 @@ describe('matchDatasetDataMarkdownImageUrls', () => { ]); }); - it('应提取 markdown 图片 URL 并忽略普通链接', () => { + it('应提取 markdown 图片 URL 并忽略普通链接', async () => { const result = matchDatasetDataMarkdownImageUrls( '![a](dataset/team/a.png) [普通链接](https://example.com) ![b](https://img.test/b.jpg)' ); @@ -487,7 +486,7 @@ describe('matchDatasetDataMarkdownImageUrls', () => { expect(result).toEqual(['dataset/team/a.png', 'https://img.test/b.jpg']); }); - it('应从多个文本字段按首次出现顺序去重图片 URL', () => { + it('应从多个文本字段按首次出现顺序去重图片 URL', async () => { const result = uniqueDatasetDataMarkdownImageUrls([ 'new ![a](dataset/team/a.png) ![a again](dataset/team/a.png)', undefined, @@ -499,7 +498,7 @@ describe('matchDatasetDataMarkdownImageUrls', () => { }); describe('getDatasetImageTrainingMode', () => { - it('有 VLM 且是图片数据时应走 imageParse', () => { + it('有 VLM 且是图片数据时应走 imageParse', async () => { expect( getDatasetImageTrainingMode({ supportVlm: true, @@ -510,7 +509,7 @@ describe('getDatasetImageTrainingMode', () => { ).toBe(TrainingModeEnum.imageParse); }); - it('有图片索引能力且正文有 markdown 图片时应走 image', () => { + it('有图片索引能力且正文有 markdown 图片时应走 image', async () => { expect( getDatasetImageTrainingMode({ supportVlm: false, @@ -520,7 +519,7 @@ describe('getDatasetImageTrainingMode', () => { ).toBe(TrainingModeEnum.image); }); - it('没有图片索引能力时应回退 chunk', () => { + it('没有图片索引能力时应回退 chunk', async () => { expect( getDatasetImageTrainingMode({ supportVlm: false, @@ -539,7 +538,7 @@ describe('getTrainingModeByCollection', () => { }; }); - it('图片自动索引有 VLM 或原生 embedding 图片索引能力时进入 image 队列', () => { + it('图片自动索引有 VLM 或原生 embedding 图片索引能力时进入 image 队列', async () => { expect( getTrainingModeByCollection({ trainingType: DatasetCollectionDataProcessModeEnum.chunk, @@ -574,7 +573,7 @@ describe('getDatasetImageIndexCapability', () => { }); }); - it('未配置 VLM 时不应自动回退到默认 VLM', () => { + it('未配置 VLM 时不应自动回退到默认 VLM', async () => { const result = getDatasetImageIndexCapability({ vectorModel: 'vision-embedding-model' }); @@ -585,7 +584,7 @@ describe('getDatasetImageIndexCapability', () => { expect(result.availableVlmModel).toBeUndefined(); }); - it('配置 VLM 时应同时返回 VLM 和多模态索引能力', () => { + it('配置 VLM 时应同时返回 VLM 和多模态索引能力', async () => { const result = getDatasetImageIndexCapability({ vectorModel: 'vision-embedding-model', vlmModel: 'dataset-vlm-model' diff --git a/projects/app/src/components/core/chat/components/FormInputResult.tsx b/projects/app/src/components/core/chat/components/FormInputResult.tsx index 467a2d75b985..9dfe40bc4203 100644 --- a/projects/app/src/components/core/chat/components/FormInputResult.tsx +++ b/projects/app/src/components/core/chat/components/FormInputResult.tsx @@ -18,7 +18,7 @@ export type FormInputResultFileItem = { * 从文件下载 URL 中解析展示用文件名。 * * FastGPT 签名下载链接通常把真实文件名放在 `filename` query 中（见 - * `/api/system/file/download/token?filename=...`），path 段往往只是 token，不可读。 + * `/api/system/file/d/alias.exp.sig`），path 段往往只是 token，不可读。 * 解析优先级：query `filename` > URL path 最后一段 > 原 URL 字符串。 * * @param url - 文件下载地址；非法 URL 时直接返回入参，避免展示层抛错。 diff --git a/projects/app/src/pages/api/core/chat/record/getCollectionQuote.ts b/projects/app/src/pages/api/core/chat/record/getCollectionQuote.ts index 2d4028a62708..3abce33d026d 100644 --- a/projects/app/src/pages/api/core/chat/record/getCollectionQuote.ts +++ b/projects/app/src/pages/api/core/chat/record/getCollectionQuote.ts @@ -135,9 +135,10 @@ async function handleInitialLoad({ .lean(); const hasMoreNext = list.length === pageSize; + const citeList = await getFormatDatasetCiteList(list); return { - list: processChatTimeFilter(getFormatDatasetCiteList(list), chatTime).map((item) => ({ + list: processChatTimeFilter(citeList, chatTime).map((item) => ({ ...item, id: item._id, anchor: item.index @@ -161,9 +162,10 @@ async function handleInitialLoad({ ); const resultList = [...prevList, centerNode, ...nextList]; + const citeList = await getFormatDatasetCiteList(resultList); return { - list: processChatTimeFilter(getFormatDatasetCiteList(resultList), chatTime).map((item) => ({ + list: processChatTimeFilter(citeList, chatTime).map((item) => ({ ...item, id: item._id, anchor: item.index @@ -192,7 +194,8 @@ async function handlePaginatedLoad({ ? await getPrevNodes(prevId, nextAnchor, pageSize, baseMatch) : await getNextNodes(nextId!, nextAnchor, pageSize, baseMatch); - const processedList = processChatTimeFilter(getFormatDatasetCiteList(list), chatTime); + const citeList = await getFormatDatasetCiteList(list); + const processedList = processChatTimeFilter(citeList, chatTime); return { list: processedList.map((item) => ({ ...item, id: item._id, anchor: item.index })), diff --git a/projects/app/src/pages/api/core/chat/record/getQuote.ts b/projects/app/src/pages/api/core/chat/record/getQuote.ts index 5c791b8ccdb8..56154419fc4a 100644 --- a/projects/app/src/pages/api/core/chat/record/getQuote.ts +++ b/projects/app/src/pages/api/core/chat/record/getQuote.ts @@ -63,7 +63,7 @@ async function handler(req: ApiRequestProps): Promise { quoteDataFieldSelector ).lean(); - const formatPreviewUrlList = getFormatDatasetCiteList(list); + const formatPreviewUrlList = await getFormatDatasetCiteList(list); const quoteList = processChatTimeFilter(formatPreviewUrlList, chatItem.time); return GetQuoteResponseSchema.parse(quoteList); diff --git a/projects/app/src/pages/api/core/dataset/collection/export.ts b/projects/app/src/pages/api/core/dataset/collection/export.ts index e70a7e4541c7..259f560f2a96 100644 --- a/projects/app/src/pages/api/core/dataset/collection/export.ts +++ b/projects/app/src/pages/api/core/dataset/collection/export.ts @@ -111,17 +111,23 @@ async function handler(req: ApiRequestProps, res: NextApiResponse) { write(`\uFEFFq,a`); - cursor.on('data', (doc) => { - const sanitizedQ = replaceS3KeyToPreviewUrl( - sanitizeCsvField(doc.q || ''), - addDays(new Date(), 90) - ); - const sanitizedA = replaceS3KeyToPreviewUrl( - sanitizeCsvField(doc.a || ''), - addDays(new Date(), 90) - ); - - write(`\n${sanitizedQ},${sanitizedA}`); + cursor.on('data', async (doc) => { + cursor.pause(); + + try { + const [sanitizedQ, sanitizedA] = await Promise.all([ + replaceS3KeyToPreviewUrl(sanitizeCsvField(doc.q || ''), addDays(new Date(), 90)), + replaceS3KeyToPreviewUrl(sanitizeCsvField(doc.a || ''), addDays(new Date(), 90)) + ]); + + write(`\n${sanitizedQ},${sanitizedA}`); + } catch (error) { + logger.error(`export usage error`, { error }); + cursor.destroy(); + return; + } + + cursor.resume(); }); cursor.on('end', () => { diff --git a/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts b/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts index 3a56025b167d..e4cf3aabdd5a 100644 --- a/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts +++ b/projects/app/src/pages/api/core/dataset/data/getQuoteData.ts @@ -57,11 +57,11 @@ async function handler(req: ApiRequestProps): Promise { return { collection, - ...formatDatasetDataValue({ + ...(await formatDatasetDataValue({ q: datasetData.q, a: datasetData.a, imageId: datasetData.imageId - }) + })) }; } else { const { datasetData, collection } = await authDatasetData({ @@ -73,11 +73,11 @@ async function handler(req: ApiRequestProps): Promise { }); return { collection, - ...formatDatasetDataValue({ + ...(await formatDatasetDataValue({ q: datasetData.q, a: datasetData.a, imageId: datasetData.imageId - }) + })) }; } })(); diff --git a/projects/app/src/pages/api/core/dataset/data/update.ts b/projects/app/src/pages/api/core/dataset/data/update.ts index 69dc6bb01239..1ff9ec6fb29d 100644 --- a/projects/app/src/pages/api/core/dataset/data/update.ts +++ b/projects/app/src/pages/api/core/dataset/data/update.ts @@ -109,9 +109,14 @@ async function handler(req: ApiRequestProps): Promise pushUpdateDataAuditLog(); + const [responseQ, responseA] = await Promise.all([ + replaceS3KeyToPreviewUrl(nextQ, addHours(new Date(), 1)), + nextA ? replaceS3KeyToPreviewUrl(nextA, addHours(new Date(), 1)) : undefined + ]); + return UpdateDatasetDataResponseSchema.parse({ - q: replaceS3KeyToPreviewUrl(nextQ, addHours(new Date(), 1)), - a: nextA ? replaceS3KeyToPreviewUrl(nextA, addHours(new Date(), 1)) : undefined + q: responseQ, + a: responseA }); } diff --git a/projects/app/src/pages/api/core/dataset/data/v2/list.ts b/projects/app/src/pages/api/core/dataset/data/v2/list.ts index 61cff1fed674..8bd768a7e793 100644 --- a/projects/app/src/pages/api/core/dataset/data/v2/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/v2/list.ts @@ -10,7 +10,6 @@ import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { addHours } from 'date-fns'; import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; -import { jwtSignS3DownloadToken } from '@fastgpt/service/common/s3/security/token'; import { replaceS3KeyToPreviewUrl } from '@fastgpt/service/core/dataset/utils'; import { GetDatasetDataListBodySchema, @@ -19,6 +18,7 @@ import { } from '@fastgpt/global/openapi/core/dataset/data/api'; import { S3Buckets } from '@fastgpt/service/common/s3/config/constants'; import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; +import { createS3DownloadAccessUrl } from '@fastgpt/service/common/s3/accessLink'; async function handler(req: ApiRequestProps): Promise { const { searchText = '', collectionId } = parseApiInput({ @@ -58,12 +58,14 @@ async function handler(req: ApiRequestProps): Promise { - item.q = replaceS3KeyToPreviewUrl(item.q, addHours(new Date(), 1)); - if (item.a) { - item.a = replaceS3KeyToPreviewUrl(item.a, addHours(new Date(), 1)); - } - }); + await Promise.all( + list.map(async (item) => { + item.q = await replaceS3KeyToPreviewUrl(item.q, addHours(new Date(), 1)); + if (item.a) { + item.a = await replaceS3KeyToPreviewUrl(item.a, addHours(new Date(), 1)); + } + }) + ); const imageIds = list.map((item) => item.imageId!).filter(Boolean); const imageSizeMap = new Map(); @@ -95,7 +97,7 @@ async function handler(req: ApiRequestProps): Promise ({ - q: replaceS3KeyToPreviewUrl(chunk.q, addDays(new Date(), 1)), - a: replaceS3KeyToPreviewUrl(chunk.a, addDays(new Date(), 1)) - })); + const chunksWithJWT = await Promise.all( + chunks.slice(0, 10).map(async (chunk) => ({ + q: await replaceS3KeyToPreviewUrl(chunk.q, addDays(new Date(), 1)), + a: await replaceS3KeyToPreviewUrl(chunk.a, addDays(new Date(), 1)) + })) + ); return GetPreviewChunksResponseSchema.parse({ chunks: chunksWithJWT, diff --git a/projects/app/src/pages/api/core/dataset/file/getSearchTestImagePreviewUrls.ts b/projects/app/src/pages/api/core/dataset/file/getSearchTestImagePreviewUrls.ts index 1fecd6bd5b95..2363bc8917ac 100644 --- a/projects/app/src/pages/api/core/dataset/file/getSearchTestImagePreviewUrls.ts +++ b/projects/app/src/pages/api/core/dataset/file/getSearchTestImagePreviewUrls.ts @@ -4,9 +4,9 @@ import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { addHours } from 'date-fns'; import { S3Buckets } from '@fastgpt/service/common/s3/config/constants'; -import { jwtSignS3DownloadToken } from '@fastgpt/service/common/s3/security/token'; import { isAuthorizedTempFileS3Key } from '@fastgpt/service/common/s3/sources/temp/key'; import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; +import { createS3DownloadAccessUrl } from '@fastgpt/service/common/s3/accessLink'; import { GetSearchTestImagePreviewUrlsBodySchema, GetSearchTestImagePreviewUrlsResponseSchema, @@ -29,16 +29,18 @@ async function handler( authApiKey: true }); - const result = keys - .filter((key) => isAuthorizedTempFileS3Key({ key, teamId })) - .map((key) => ({ - key, - previewUrl: jwtSignS3DownloadToken({ - objectKey: key, - bucketName: S3Buckets.private, - expiredTime: addHours(new Date(), 1) - }) - })); + const result = await Promise.all( + keys + .filter((key) => isAuthorizedTempFileS3Key({ key, teamId })) + .map(async (key) => ({ + key, + previewUrl: await createS3DownloadAccessUrl({ + objectKey: key, + bucketName: S3Buckets.private, + expiredTime: addHours(new Date(), 1) + }) + })) + ); return GetSearchTestImagePreviewUrlsResponseSchema.parse(result); } diff --git a/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts b/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts index 40481ce28d7e..0f7252629ccc 100644 --- a/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts +++ b/projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts @@ -4,7 +4,6 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/datas import { NextAPI } from '@/service/middleware/entry'; import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; -import { jwtSignS3DownloadToken } from '@fastgpt/service/common/s3/security/token'; import { addMinutes } from 'date-fns'; import { GetTrainingDataDetailBodySchema, @@ -13,6 +12,7 @@ import { } from '@fastgpt/global/openapi/core/dataset/training/api'; import { S3Buckets } from '@fastgpt/service/common/s3/config/constants'; import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; +import { createS3DownloadAccessUrl } from '@fastgpt/service/common/s3/accessLink'; async function handler(req: ApiRequestProps): Promise { const { collectionId, dataId } = parseApiInput({ @@ -39,19 +39,21 @@ async function handler(req: ApiRequestProps): Promise { - if (!filename) return ''; - try { - return decodeURIComponent(filename); - } catch { - return filename; +async function handler(req: ApiRequestProps, res: NextApiResponse) { + if (!['GET', 'HEAD'].includes(req.method || '')) { + return jsonRes(res, { code: 405, error: 'Method not allowed' }); } -}; -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - if (!['GET', 'HEAD'].includes(req.method || '')) { - return Promise.reject('Method not allowed'); - } + const { token, filename } = parseApiInput({ + req, + querySchema: S3JwtDownloadRouteQuerySchema + }).query; - const { token, filename: queryFilename } = req.query as { token: string; filename?: string }; + try { const { objectKey, bucketName } = await jwtVerifyS3DownloadToken(token); - const bucket = global.s3BucketMap[bucketName]; - if (!bucket) { - return jsonRes(res, { - code: 404, - error: 'S3 bucket not found' - }); - } - - const [stream, metadata] = await Promise.all([ - bucket.getFileStream(objectKey), - bucket.getFileMetadata(objectKey) - ]); - - if (!stream) { - return jsonRes(res, { - code: 404, - error: 'File not found' - }); - } - - if (metadata?.contentType) { - res.setHeader( - 'Content-Type', - ensureTextContentTypeCharset({ - contentType: metadata.contentType, - filename: metadata.filename - }) - ); - } - if (metadata?.contentLength) { - res.setHeader('Content-Length', metadata.contentLength); - } - const filename = - parseRequestFilename(queryFilename) || - metadata?.filename || - path.basename(objectKey) || - 'file'; - res.setHeader('Content-Disposition', getContentDisposition({ filename, type: 'inline' })); - res.setHeader('Cache-Control', 'public, max-age=31536000'); - - if (req.method === 'HEAD') { - res.status(200).end(); - return; - } - - stream.pipe(res); - - stream.on('error', (error) => { - logger.error('Error reading proxy download stream', { + return await handleS3ProxyDownload({ + req, + res, + payload: { objectKey, bucketName, - error - }); - if (!res.headersSent) { - jsonRes(res, { - code: 500, - error - }); + filename } }); - stream.on('end', () => { - res.end(); - }); } catch (error) { - jsonRes(res, { - code: 500, - error - }); + return handleS3ProxyRouteError({ res, error }); } } + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/system/file/u/[token].ts b/projects/app/src/pages/api/system/file/u/[token].ts new file mode 100644 index 000000000000..09968c51aa8d --- /dev/null +++ b/projects/app/src/pages/api/system/file/u/[token].ts @@ -0,0 +1,37 @@ +import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import type { NextApiResponse } from 'next'; +import { NextAPI } from '@/service/middleware/entry'; +import { + S3UploadAccessRouteQuerySchema, + verifyS3UploadSessionToken +} from '@fastgpt/service/common/s3/accessLink'; +import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; +import { handleS3ProxyRouteError, handleS3ProxyUpload } from '@/service/common/s3/proxy'; +import { jsonRes } from '@fastgpt/service/common/response'; + +async function handler(req: ApiRequestProps, res: NextApiResponse) { + if (req.method !== 'PUT') { + return jsonRes(res, { code: 405, error: 'Method not allowed' }); + } + + const { token } = parseApiInput({ + req, + querySchema: S3UploadAccessRouteQuerySchema + }).query; + + try { + const payload = await verifyS3UploadSessionToken(token); + + return await handleS3ProxyUpload({ req, payload }); + } catch (error) { + return handleS3ProxyRouteError({ res, error }); + } +} + +export default NextAPI(handler); + +export const config = { + api: { + bodyParser: false + } +}; diff --git a/projects/app/src/pages/api/system/file/upload/[token].ts b/projects/app/src/pages/api/system/file/upload/[token].ts index c0d5ff62b345..4ce1e40e6be8 100644 --- a/projects/app/src/pages/api/system/file/upload/[token].ts +++ b/projects/app/src/pages/api/system/file/upload/[token].ts @@ -1,207 +1,43 @@ -import type { NextApiRequest } from 'next'; -import { Transform } from 'node:stream'; -import { getContentDisposition } from '@fastgpt/global/common/file/tools'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; +import type { NextApiResponse } from 'next'; +import z from 'zod'; import { NextAPI } from '@/service/middleware/entry'; import { jwtVerifyS3UploadToken } from '@fastgpt/service/common/s3/security/token'; -import type { UploadConstraints } from '@fastgpt/service/common/s3/contracts/type'; -import { - getUploadInspectBytes, - validateUploadFile -} from '@fastgpt/service/common/s3/validation/upload'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { handleS3ProxyRouteError, handleS3ProxyUpload } from '@/service/common/s3/proxy'; +import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; -type GuardStreamOptions = { - maxSize: number; - uploadConstraints: UploadConstraints; - filename?: string; -}; - -type ValidatedUploadFile = Awaited>; - -const parseContentLength = (value: string | string[] | undefined) => { - const raw = Array.isArray(value) ? value[0] : value; - if (!raw) return undefined; - - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return undefined; - return parsed; -}; - -const createUploadGuardStream = ({ maxSize, uploadConstraints, filename }: GuardStreamOptions) => { - const inspectBytes = getUploadInspectBytes(filename); - let uploadedBytes = 0; - let bufferedBytes = 0; - const chunks: Buffer[] = []; - let validatedFile: ValidatedUploadFile | undefined; - let validationSettled = false; - let resolveValidatedUpload!: (value: ValidatedUploadFile) => void; - let rejectValidatedUpload!: (reason?: unknown) => void; - - const validatedUpload = new Promise((resolve, reject) => { - resolveValidatedUpload = resolve; - rejectValidatedUpload = reject; - }); - - const settleValidation = ({ - result, - error - }: { - result?: ValidatedUploadFile; - error?: unknown; - }) => { - if (validationSettled) return; - validationSettled = true; - - if (error) { - rejectValidatedUpload(error); - return; - } - if (result) { - resolveValidatedUpload(result); - } - }; - - const validateBuffer = async () => { - if (validatedFile) return validatedFile; - const buffer = Buffer.concat(chunks, bufferedBytes); - - const result = await validateUploadFile({ - buffer, - filename, - uploadConstraints - }); - - validatedFile = result; - settleValidation({ result }); - - return result; - }; - - const stream = new Transform({ - transform(chunk, _, callback) { - uploadedBytes += chunk.length; - if (uploadedBytes > maxSize) { - const error = new Error('EntityTooLarge'); - settleValidation({ error }); - callback(error); - return; - } - - if (validatedFile) { - callback(null, chunk); - return; - } - - chunks.push(chunk); - bufferedBytes += chunk.length; +const S3JwtUploadRouteQuerySchema = z.object({ + token: z.string().min(1) +}); - if (bufferedBytes < inspectBytes) { - callback(); - return; - } - - validateBuffer() - .then(() => { - const initialBuffer = Buffer.concat(chunks, bufferedBytes); - chunks.length = 0; - bufferedBytes = 0; - callback(null, initialBuffer); - }) - .catch((error) => { - settleValidation({ error }); - callback(error); - }); - }, - flush(callback) { - validateBuffer() - .then(() => { - if (bufferedBytes > 0) { - callback(null, Buffer.concat(chunks, bufferedBytes)); - return; - } - callback(); - }) - .catch((error) => { - settleValidation({ error }); - callback(error); - }); - } - }); - - stream.once('error', (error) => { - settleValidation({ error }); - }); - stream.once('close', () => { - if (!validationSettled && !validatedFile) { - settleValidation({ error: new Error('UploadStreamClosed') }); - } - }); - - return { - stream, - validatedUpload - }; -}; - -const buildUploadMetadata = ({ - metadata, - filename -}: { - metadata?: Record; - filename?: string; -}) => { - if (!filename) return metadata; - - return { - ...metadata, - contentDisposition: getContentDisposition({ - filename, - type: 'attachment' - }), - originFilename: encodeURIComponent(filename) - }; -}; - -async function handler(req: NextApiRequest) { +async function handler(req: ApiRequestProps, res: NextApiResponse) { if (req.method !== 'PUT') { - return Promise.reject('Method not allowed'); + return jsonRes(res, { code: 405, error: 'Method not allowed' }); } - const { token } = req.query as { token: string }; - const { objectKey, bucketName, maxSize, uploadConstraints, metadata } = - await jwtVerifyS3UploadToken(token); - - const bucket = global.s3BucketMap[bucketName]; - if (!bucket) { - return Promise.reject('S3 bucket not found'); - } - - const contentLength = parseContentLength(req.headers['content-length']); - if (contentLength && contentLength > maxSize) { - return Promise.reject('EntityTooLarge'); + const { token } = parseApiInput({ + req, + querySchema: S3JwtUploadRouteQuerySchema + }).query; + + try { + const { objectKey, bucketName, maxSize, uploadConstraints, metadata } = + await jwtVerifyS3UploadToken(token); + + return await handleS3ProxyUpload({ + req, + payload: { + objectKey, + bucketName, + maxSize, + uploadConstraints, + metadata + } + }); + } catch (error) { + return handleS3ProxyRouteError({ res, error }); } - - const filename = metadata?.originFilename; - const { stream: guardStream, validatedUpload } = createUploadGuardStream({ - maxSize, - uploadConstraints, - filename - }); - - req.pipe(guardStream); - const validatedFile = await validatedUpload; - - await bucket.client.uploadObject({ - key: objectKey, - body: guardStream, - contentType: validatedFile.contentType, - contentLength, - metadata: buildUploadMetadata({ - metadata, - filename: validatedFile.filename - }) - }); - - return { success: true }; } export default NextAPI(handler); diff --git a/projects/app/src/service/common/s3/proxy.ts b/projects/app/src/service/common/s3/proxy.ts new file mode 100644 index 000000000000..2d8b17634f97 --- /dev/null +++ b/projects/app/src/service/common/s3/proxy.ts @@ -0,0 +1,402 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { Transform } from 'node:stream'; +import path from 'node:path'; +import { getContentDisposition } from '@fastgpt/global/common/file/tools'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { ERROR_RESPONSE } from '@fastgpt/global/common/error/errorCode'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; +import { isS3AccessLinkError } from '@fastgpt/service/common/s3/accessLink'; +import type { + S3ProxyDownloadPayload, + S3ProxyUploadPayload +} from '@fastgpt/service/common/s3/accessLink'; +import type { UploadConstraints } from '@fastgpt/service/common/s3/contracts/type'; +import { + DEFAULT_CONTENT_TYPE, + ensureTextContentTypeCharset +} from '@fastgpt/service/common/s3/utils/mime'; +import { + getUploadInspectBytes, + validateUploadFile +} from '@fastgpt/service/common/s3/validation/upload'; + +const logger = getLogger(LogCategories.INFRA.FILE); + +type GuardStreamOptions = { + maxSize: number; + uploadConstraints: UploadConstraints; + filename?: string; +}; + +type ValidatedUploadFile = Awaited>; + +const parseRequestFilename = (filename?: string) => { + if (!filename) return ''; + + try { + return decodeURIComponent(filename); + } catch { + return filename; + } +}; + +export const parseS3ProxyContentLength = (value: string | string[] | undefined) => { + const raw = Array.isArray(value) ? value[0] : value; + if (!raw) return undefined; + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return undefined; + return parsed; +}; + +const createUploadGuardStream = ({ maxSize, uploadConstraints, filename }: GuardStreamOptions) => { + const inspectBytes = getUploadInspectBytes(filename); + let uploadedBytes = 0; + let bufferedBytes = 0; + const chunks: Buffer[] = []; + let validatedFile: ValidatedUploadFile | undefined; + let validationSettled = false; + let resolveValidatedUpload!: (value: ValidatedUploadFile) => void; + let rejectValidatedUpload!: (reason?: unknown) => void; + + const validatedUpload = new Promise((resolve, reject) => { + resolveValidatedUpload = resolve; + rejectValidatedUpload = reject; + }); + + const settleValidation = ({ + result, + error + }: { + result?: ValidatedUploadFile; + error?: unknown; + }) => { + if (validationSettled) return; + validationSettled = true; + + if (error) { + rejectValidatedUpload(error); + return; + } + if (result) { + resolveValidatedUpload(result); + } + }; + + const validateBuffer = async () => { + if (validatedFile) return validatedFile; + const buffer = Buffer.concat(chunks, bufferedBytes); + + const result = await validateUploadFile({ + buffer, + filename, + uploadConstraints + }); + + validatedFile = result; + settleValidation({ result }); + + return result; + }; + + const stream = new Transform({ + transform(chunk, _, callback) { + uploadedBytes += chunk.length; + if (uploadedBytes > maxSize) { + const error = new Error('EntityTooLarge'); + settleValidation({ error }); + callback(error); + return; + } + + if (validatedFile) { + callback(null, chunk); + return; + } + + chunks.push(chunk); + bufferedBytes += chunk.length; + + if (bufferedBytes < inspectBytes) { + callback(); + return; + } + + validateBuffer() + .then(() => { + const initialBuffer = Buffer.concat(chunks, bufferedBytes); + chunks.length = 0; + bufferedBytes = 0; + callback(null, initialBuffer); + }) + .catch((error) => { + settleValidation({ error }); + callback(error); + }); + }, + flush(callback) { + validateBuffer() + .then(() => { + if (bufferedBytes > 0) { + callback(null, Buffer.concat(chunks, bufferedBytes)); + return; + } + callback(); + }) + .catch((error) => { + settleValidation({ error }); + callback(error); + }); + } + }); + + stream.once('error', (error) => { + settleValidation({ error }); + }); + stream.once('close', () => { + if (!validationSettled && !validatedFile) { + settleValidation({ error: new Error('UploadStreamClosed') }); + } + }); + + return { + stream, + validatedUpload + }; +}; + +export const buildS3UploadMetadata = ({ + metadata, + filename +}: { + metadata?: Record; + filename?: string; +}) => { + if (!filename) return metadata; + + return { + ...metadata, + contentDisposition: getContentDisposition({ + filename, + type: 'attachment' + }), + originFilename: encodeURIComponent(filename) + }; +}; + +/** + * 代理输出 S3 对象下载流。 + * + * 该函数只处理文件流、响应头和对象不存在等存储层错误；业务权限和短链校验必须在调用前完成。 + */ +export const handleS3ProxyDownload = async ({ + req, + res, + payload +}: { + req: NextApiRequest; + res: NextApiResponse; + payload: S3ProxyDownloadPayload; +}) => { + const { objectKey, bucketName } = payload; + const bucket = global.s3BucketMap[bucketName]; + + if (!bucket) { + throw new Error('S3 bucket not found'); + } + + const [stream, metadata] = await Promise.all([ + bucket.getFileStream(objectKey), + bucket.getFileMetadata(objectKey) + ]); + + if (!stream) { + throw CommonErrEnum.fileNotFound; + } + + const filename = + parseRequestFilename(payload.filename) || + metadata?.filename || + path.basename(objectKey) || + 'file'; + const contentType = payload.responseContentType || metadata?.contentType || DEFAULT_CONTENT_TYPE; + res.setHeader( + 'Content-Type', + ensureTextContentTypeCharset({ + contentType, + filename + }) + ); + if (metadata?.contentLength) { + res.setHeader('Content-Length', metadata.contentLength); + } + res.setHeader('Content-Disposition', getContentDisposition({ filename, type: 'inline' })); + res.setHeader('Cache-Control', 'public, max-age=31536000'); + + if (req.method === 'HEAD') { + res.status(200).end(); + return; + } + + stream.on('error', (error) => { + logger.error('Error reading proxy download stream', { + objectKey, + bucketName, + error + }); + + if (!res.headersSent) { + jsonRes(res, { + code: 500, + error + }); + return; + } + + res.destroy(error); + }); + stream.on('end', () => { + res.end(); + }); + + stream.pipe(res); +}; + +/** + * 代理处理上传请求。 + * + * 上传内容会先经过大小和文件类型 guard，再写入 S3。该函数不负责签发或校验上传 session。 + */ +export const handleS3ProxyUpload = async ({ + req, + payload +}: { + req: NextApiRequest; + payload: S3ProxyUploadPayload; +}) => { + const { objectKey, bucketName, maxSize, uploadConstraints, metadata } = payload; + const bucket = global.s3BucketMap[bucketName]; + + if (!bucket) { + throw new Error('S3 bucket not found'); + } + + const contentLength = parseS3ProxyContentLength(req.headers['content-length']); + if (contentLength && contentLength > maxSize) { + throw new Error('EntityTooLarge'); + } + + const filename = metadata?.originFilename; + const { stream: guardStream, validatedUpload } = createUploadGuardStream({ + maxSize, + uploadConstraints, + filename + }); + + req.pipe(guardStream); + const validatedFile = await validatedUpload; + + await bucket.client.uploadObject({ + key: objectKey, + body: guardStream, + contentType: validatedFile.contentType, + contentLength, + metadata: buildS3UploadMetadata({ + metadata, + filename: validatedFile.filename + }) + }); + + return { success: true }; +}; + +const getProxyErrorKey = (error: unknown) => { + if (typeof error === 'string') return error; + if (error instanceof Error) return error.message; + return undefined; +}; + +/** + * 把文件代理层可预期的错误映射成稳定的 HTTP 状态和公开错误。 + * + * accessLink 内部错误统一伪装成未授权文件；旧 JWT、上传类型校验和对象缺失则保留既有 + * 业务错误，避免短链路由改造后把客户端错误误报成 500。 + */ +export function resolveS3ProxyErrorResponse(error: unknown): { + httpStatus: number; + publicError: unknown; +} { + if (isS3AccessLinkError(error)) { + return { + httpStatus: 403, + publicError: CommonErrEnum.unAuthFile + }; + } + + const errorKey = getProxyErrorKey(error); + + if (errorKey === CommonErrEnum.unAuthFile) { + return { + httpStatus: 403, + publicError: CommonErrEnum.unAuthFile + }; + } + + if (errorKey === CommonErrEnum.fileNotFound) { + return { + httpStatus: 404, + publicError: CommonErrEnum.fileNotFound + }; + } + + if (errorKey === 'S3 bucket not found') { + return { + httpStatus: 500, + publicError: error + }; + } + + if (errorKey === 'EntityTooLarge') { + return { + httpStatus: 413, + publicError: error + }; + } + + const errorResponse = errorKey ? ERROR_RESPONSE[errorKey] : undefined; + if ( + typeof errorResponse?.code === 'number' && + errorResponse.code >= 510000 && + errorResponse.code < 511000 + ) { + return { + httpStatus: 400, + publicError: error + }; + } + + return { + httpStatus: 500, + publicError: error + }; +} + +/** + * 文件代理 API 的统一错误出口。 + * + * 对已经开始流式输出的响应，只结束连接；否则交给 `jsonRes` 保持统一响应结构。 + */ +export function handleS3ProxyRouteError({ res, error }: { res: NextApiResponse; error: unknown }) { + if (res.headersSent) { + res.end(); + return; + } + + const { httpStatus, publicError } = resolveS3ProxyErrorResponse(error); + + return jsonRes(res, { + code: httpStatus, + error: publicError + }); +} diff --git a/projects/app/test/api/system/file/accessLink.test.ts b/projects/app/test/api/system/file/accessLink.test.ts new file mode 100644 index 000000000000..9e67b3399e43 --- /dev/null +++ b/projects/app/test/api/system/file/accessLink.test.ts @@ -0,0 +1,400 @@ +import { Readable } from 'node:stream'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { ERROR_RESPONSE } from '@fastgpt/global/common/error/errorCode'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { S3ErrEnum } from '@fastgpt/global/common/error/code/s3'; +import { parseS3UploadError } from '@fastgpt/global/common/error/s3'; +import { jsonRes } from '@fastgpt/service/common/response'; +import downloadAccessHandler from '@/pages/api/system/file/d/[signedAlias]'; +import uploadAccessHandler from '@/pages/api/system/file/u/[token]'; +import { + createS3DownloadAccessUrl, + createS3UploadAccessUrl, + encodeExpiresAtMinute, + revokeS3UploadSessionToken, + signS3DownloadAlias +} from '@fastgpt/service/common/s3/accessLink'; + +const makeMockRes = () => { + const headers: Record = {}; + const res = { + headers, + statusCode: 200, + headersSent: false, + writableFinished: false, + body: undefined as any, + setHeader: vi.fn((key: string, value: string | number) => { + headers[key] = value; + }), + getHeader: vi.fn((key: string) => headers[key]), + once: vi.fn(() => res), + status: vi.fn((statusCode: number) => { + res.statusCode = statusCode; + return res; + }), + end: vi.fn(() => { + res.writableFinished = true; + res.headersSent = true; + }), + json: vi.fn((body: any) => { + res.body = body; + res.writableFinished = true; + res.headersSent = true; + return res; + }), + destroy: vi.fn() + }; + return res; +}; + +const makeMockStream = () => { + const stream = { + pipe: vi.fn(), + on: vi.fn(() => stream) + }; + return stream; +}; + +const createDownloadReq = (signedAlias: string) => + ({ + method: 'GET', + url: `/api/system/file/d/${signedAlias}`, + headers: {}, + query: { signedAlias } + }) as any; + +const createUploadReq = ({ + token, + contentLength, + chunks = [] +}: { + token: string; + contentLength?: string; + chunks?: Buffer[]; +}) => + ({ + method: 'PUT', + url: `/api/system/file/u/${token}`, + headers: { + ...(contentLength ? { 'content-length': contentLength } : {}) + }, + query: { token }, + pipe: vi.fn((target) => { + Readable.from(chunks).pipe(target); + return target; + }) + }) as any; + +const getFutureDate = (minutes: number) => new Date(Date.now() + minutes * 60 * 1000); + +const extractLastPathSegment = (url: string) => url.split('/').pop() || ''; + +const buildSignedAlias = ({ + aliasId = 'R7mQG0Yh2kVxP9Za', + expiresAt = getFutureDate(10) +}: { + aliasId?: string; + expiresAt?: Date; +} = {}) => { + const expMinute36 = encodeExpiresAtMinute(expiresAt); + const sig = signS3DownloadAlias({ aliasId, expMinute36 }); + + return `${aliasId}.${expMinute36}.${sig}`; +}; + +const createTranslator = () => + vi.fn((key: string, params?: Record) => + params ? `${key}:${JSON.stringify(params)}` : key + ); + +const setupJsonResMock = () => { + vi.mocked(jsonRes).mockImplementation((res: any, props: any = {}) => { + const { code = 200, data = null, error, message = '' } = props; + + if (!error) { + return res.status(code).json({ + code, + statusText: '', + message, + data + }); + } + + const errorKey = typeof error === 'string' ? error : error?.message; + const errorResponse = errorKey ? ERROR_RESPONSE[errorKey] : undefined; + const httpStatus = (() => { + if (errorKey === 'EntityTooLarge') return 413; + if ( + typeof errorResponse?.code === 'number' && + errorResponse.code >= 510000 && + errorResponse.code < 511000 + ) { + return 400; + } + if (typeof code === 'number' && code >= 400 && code <= 599) return code; + return 500; + })(); + + return res.status(httpStatus).json({ + code: errorResponse?.code ?? code, + statusText: errorResponse?.statusText ?? 'error', + message: message || errorResponse?.message || errorKey || '请求错误', + data: errorResponse?.data ?? null + }); + }); +}; + +describe('s3 short access link api', () => { + beforeEach(() => { + vi.clearAllMocks(); + setupJsonResMock(); + global.s3BucketMap = {} as any; + }); + + it('returns 403 for expired signed aliases', async () => { + const signedAlias = buildSignedAlias({ expiresAt: getFutureDate(-10) }); + const res = makeMockRes() as any; + + await downloadAccessHandler(createDownloadReq(signedAlias), res); + + expect(res.statusCode).toBe(403); + expect(res.body.statusText).toBe(CommonErrEnum.unAuthFile); + }); + + it('returns 403 for tampered signed alias expiry', async () => { + const aliasId = 'R7mQG0Yh2kVxP9Za'; + const expMinute36 = encodeExpiresAtMinute(getFutureDate(10)); + const sig = signS3DownloadAlias({ aliasId, expMinute36 }); + const tamperedExpMinute36 = (Number.parseInt(expMinute36, 36) + 60).toString(36); + const res = makeMockRes() as any; + + await downloadAccessHandler(createDownloadReq(`${aliasId}.${tamperedExpMinute36}.${sig}`), res); + + expect(res.statusCode).toBe(403); + expect(res.body.statusText).toBe(CommonErrEnum.unAuthFile); + }); + + it('returns 403 when the signed alias has no backing document', async () => { + const res = makeMockRes() as any; + + await downloadAccessHandler(createDownloadReq(buildSignedAlias()), res); + + expect(res.statusCode).toBe(403); + expect(res.body.statusText).toBe(CommonErrEnum.unAuthFile); + }); + + it('returns 404 when a valid alias points to a missing S3 object', async () => { + const url = await createS3DownloadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'dataset/team/missing.png', + expiredTime: getFutureDate(10), + filename: 'missing.png' + }); + const signedAlias = extractLastPathSegment(url); + global.s3BucketMap = { + 'fastgpt-private': { + getFileStream: vi.fn().mockResolvedValue(null), + getFileMetadata: vi.fn().mockResolvedValue(undefined) + } + } as any; + const res = makeMockRes() as any; + + await downloadAccessHandler(createDownloadReq(signedAlias), res); + + expect(res.statusCode).toBe(404); + expect(res.body.statusText).toBe(CommonErrEnum.fileNotFound); + }); + + it('streams a valid short download alias with S3 metadata headers', async () => { + const url = await createS3DownloadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'dataset/team/page.md', + expiredTime: getFutureDate(10), + filename: 'page.md' + }); + const signedAlias = extractLastPathSegment(url); + const stream = makeMockStream(); + global.s3BucketMap = { + 'fastgpt-private': { + getFileStream: vi.fn().mockResolvedValue(stream), + getFileMetadata: vi.fn().mockResolvedValue({ + filename: 'page.md', + contentType: 'text/markdown', + contentLength: 5 + }) + } + } as any; + const res = makeMockRes() as any; + + await downloadAccessHandler(createDownloadReq(signedAlias), res); + + expect(res.headers['Content-Type']).toBe('text/markdown; charset=utf-8'); + expect(res.headers['Content-Length']).toBe(5); + expect(stream.pipe).toHaveBeenCalledWith(res); + }); + + it('returns 403 for missing, expired, or revoked upload tokens', async () => { + const expiredUrl = await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/expired.txt', + expiredTime: getFutureDate(-10), + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'text/plain' + } + }); + const revokedUrl = await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/revoked.txt', + expiredTime: getFutureDate(10), + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'text/plain' + } + }); + const cases = [ + 'wZ6rm5X4l6Ygm1oDQX8JbA', + extractLastPathSegment(expiredUrl), + extractLastPathSegment(revokedUrl) + ]; + await revokeS3UploadSessionToken(extractLastPathSegment(revokedUrl)); + + for (const token of cases) { + const res = makeMockRes() as any; + + await uploadAccessHandler(createUploadReq({ token, contentLength: '1' }), res); + + expect(res.statusCode).toBe(403); + expect(res.body.statusText).toBe(CommonErrEnum.unAuthFile); + } + }); + + it('returns 413 for upload bodies larger than the session max size', async () => { + const url = await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/large.txt', + expiredTime: getFutureDate(10), + maxSize: 1, + uploadConstraints: { + defaultContentType: 'text/plain' + } + }); + const res = makeMockRes() as any; + global.s3BucketMap = { + 'fastgpt-private': { + client: { + uploadObject: vi.fn() + } + } + } as any; + + await uploadAccessHandler( + createUploadReq({ + token: extractLastPathSegment(url), + contentLength: '2' + }), + res + ); + + const t = createTranslator(); + expect(res.statusCode).toBe(413); + expect( + parseS3UploadError({ + t, + error: { response: { data: res.body } }, + maxSize: 1 + }) + ).toContain('common:error:s3_upload_file_too_large'); + }); + + it('returns 400 for upload file type validation errors', async () => { + const url = await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/file.png', + expiredTime: getFutureDate(10), + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'image/png', + allowedExtensions: ['.png'] + }, + metadata: { + originFilename: encodeURIComponent('file.png') + } + }); + const uploadObject = vi.fn(); + global.s3BucketMap = { + 'fastgpt-private': { + client: { + uploadObject + } + } + } as any; + const res = makeMockRes() as any; + + await uploadAccessHandler( + createUploadReq({ + token: extractLastPathSegment(url), + contentLength: '5', + chunks: [Buffer.from('hello')] + }), + res + ); + + const t = createTranslator(); + expect(res.statusCode).toBe(400); + expect([S3ErrEnum.invalidUploadFileType, S3ErrEnum.uploadFileTypeMismatch]).toContain( + res.body.statusText + ); + expect( + parseS3UploadError({ + t, + error: { response: { data: res.body } } + }) + ).toBe('common:error:s3_upload_invalid_file_type'); + expect(uploadObject).not.toHaveBeenCalled(); + }); + + it('uploads a valid short upload session through the proxy handler', async () => { + const url = await createS3UploadAccessUrl({ + bucketName: 'fastgpt-private', + objectKey: 'chat/app/user/chat/file.txt', + expiredTime: getFutureDate(10), + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'text/plain', + allowedExtensions: ['.txt'] + }, + metadata: { + originFilename: encodeURIComponent('file.txt') + } + }); + const uploadObject = vi.fn().mockResolvedValue(undefined); + global.s3BucketMap = { + 'fastgpt-private': { + client: { + uploadObject + } + } + } as any; + + await uploadAccessHandler( + createUploadReq({ + token: extractLastPathSegment(url), + contentLength: '5', + chunks: [Buffer.from('hello')] + }), + makeMockRes() as any + ); + + expect(uploadObject).toHaveBeenCalledWith( + expect.objectContaining({ + key: 'chat/app/user/chat/file.txt', + contentType: 'text/plain', + contentLength: 5, + metadata: expect.objectContaining({ + originFilename: 'file.txt' + }) + }) + ); + }); +}); diff --git a/projects/app/test/api/system/file/sourceContentType.test.ts b/projects/app/test/api/system/file/sourceContentType.test.ts index 51c807bee309..4e639ba67327 100644 --- a/projects/app/test/api/system/file/sourceContentType.test.ts +++ b/projects/app/test/api/system/file/sourceContentType.test.ts @@ -1,12 +1,21 @@ +import { Readable } from 'node:stream'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import proxyDownloadHandler from '@/pages/api/system/file/download/[token]'; +import proxyUploadHandler from '@/pages/api/system/file/upload/[token]'; import legacyFileHandler from '@/pages/api/system/file/[jwt]'; -import { jwtVerifyS3DownloadToken, verifyToken } from '@fastgpt/service/common/s3/security/token'; +import { resolveS3ProxyErrorResponse } from '@/service/common/s3/proxy'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { + jwtVerifyS3DownloadToken, + jwtVerifyS3UploadToken, + verifyToken +} from '@fastgpt/service/common/s3/security/token'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { getS3ChatSource } from '@fastgpt/service/common/s3/sources/chat'; vi.mock('@fastgpt/service/common/s3/security/token', () => ({ jwtVerifyS3DownloadToken: vi.fn(), + jwtVerifyS3UploadToken: vi.fn(), verifyToken: vi.fn(), isS3ObjectKeyTokenPayload: vi.fn() })); @@ -31,13 +40,25 @@ const makeMockRes = () => { const headers: Record = {}; const res = { headers, + statusCode: 200, headersSent: false, + writableFinished: false, setHeader: vi.fn((key: string, value: string | number) => { headers[key] = value; }), - status: vi.fn(() => res), - end: vi.fn(), - json: vi.fn() + getHeader: vi.fn((key: string) => headers[key]), + once: vi.fn(() => res), + status: vi.fn((statusCode: number) => { + res.statusCode = statusCode; + return res; + }), + end: vi.fn(() => { + res.writableFinished = true; + }), + json: vi.fn(() => { + res.writableFinished = true; + }), + destroy: vi.fn() }; return res; }; @@ -67,7 +88,12 @@ describe('system file response content type', () => { type: 'download' }); - const req = { method: 'GET', query: { token: 'token' } } as any; + const req = { + method: 'GET', + url: '/api/system/file/download/token', + headers: {}, + query: { token: 'token' } + } as any; const res = makeMockRes() as any; await proxyDownloadHandler(req, res); @@ -96,7 +122,12 @@ describe('system file response content type', () => { type: 'download' }); - const req = { method: 'GET', query: { token: 'token' } } as any; + const req = { + method: 'GET', + url: '/api/system/file/download/token', + headers: {}, + query: { token: 'token' } + } as any; const res = makeMockRes() as any; await proxyDownloadHandler(req, res); @@ -104,6 +135,61 @@ describe('system file response content type', () => { expect(res.headers['Content-Type']).toBe('image/png'); }); + it('maps expected proxy errors without turning them into HTTP 500', () => { + expect(resolveS3ProxyErrorResponse(CommonErrEnum.unAuthFile)).toEqual({ + httpStatus: 403, + publicError: CommonErrEnum.unAuthFile + }); + expect(resolveS3ProxyErrorResponse(new Error('InvalidUploadFileType')).httpStatus).toBe(400); + expect(resolveS3ProxyErrorResponse(new Error('EntityTooLarge')).httpStatus).toBe(413); + }); + + it('keeps legacy jwt proxy upload mode working through the shared proxy handler', async () => { + const uploadObject = vi.fn().mockResolvedValue(undefined); + (global as any).s3BucketMap = { + 'fastgpt-private': { + client: { + uploadObject + } + } + }; + vi.mocked(jwtVerifyS3UploadToken).mockResolvedValue({ + objectKey: 'chat/app/user/chat/file.txt', + bucketName: 'fastgpt-private', + maxSize: 1024, + uploadConstraints: { + defaultContentType: 'text/plain', + allowedExtensions: ['.txt'] + }, + metadata: { + originFilename: encodeURIComponent('file.txt') + }, + type: 'upload' + }); + + const req = { + method: 'PUT', + url: '/api/system/file/upload/token', + headers: { 'content-length': '5' }, + query: { token: 'token' }, + pipe: vi.fn((target) => { + Readable.from([Buffer.from('hello')]).pipe(target); + return target; + }) + } as any; + const res = makeMockRes() as any; + + await proxyUploadHandler(req, res); + + expect(uploadObject).toHaveBeenCalledWith( + expect.objectContaining({ + key: 'chat/app/user/chat/file.txt', + contentType: 'text/plain', + contentLength: 5 + }) + ); + }); + it('adds utf-8 charset for text files in legacy file entry', async () => { const stream = makeMockStream(); const datasetSource = {