Xhs crawler

packages/server/package.json

@@ -7,7 +7,7 @@
   "types": "dist/index.d.ts",
   "scripts": {
     "start": "node dist",
-    "build": "rimraf dist/** && tsc && cp .env dist"
+    "build": "rimraf dist/** && tsc && cp .env dist && cp -R src/libs dist/"
   },
   "dependencies": {
     "@encrejs/api": "^0.0.3",
@@ -19,6 +19,7 @@
     "express-rate-limit": "^6.7.0",
     "express-response-size": "^0.0.3",
     "http-status-codes": "^2.3.0",
+    "node-cache": "^5.1.2",
     "p-retry": "^6.2.0",
     "playwright": "^1.49.1",
     "sharp": "^0.33.5",

packages/server/src/controllers/crawlers/weibo/index.ts

@@ -101,6 +101,12 @@ export async function getAllNotes(req: Request, res: Response): Promise<void> {
   } catch (error) {
     res.status(500).json({ error: `获取用户帖子失败: ${(error as Error).message}` });
   }
+  // try{
+  //   const creatorInfo = await weiboService.getCreatorInfoByID(creatorId);
+  //   res.json({creatorinfo: creatorInfo});
+  // } catch (error) {
+  //   res.status(500).json({ error: `获取用户详情失败: ${(error as Error).message}` });
+  // }
 }
 
 

packages/server/src/controllers/crawlers/xhs/index.ts

@@ -0,0 +1,78 @@
+import { Request, Response } from 'express';
+import { xhsService } from '../../../services/crawlers/xhs/index.js';
+import { logger } from '../../../utils/crawlers/logger.js';
+
+const XiaoHongshuService = new xhsService();
+
+export async function start(req: Request, res: Response): Promise<void> {
+  const {loginType, cookieStr, task, creatorId } = req.body;
+
+  if (!loginType) {
+    res.status(400).json({ error: '登录方式 (loginType) 是必填的' });
+    return;
+  }
+
+  // if (!task) {
+  //   res.status(400).json({ error: '任务类型 (task) 是必填的' });
+  //   return;
+  // }
+
+
+  try {
+    const creatorData = await XiaoHongshuService.start(loginType, cookieStr, task, creatorId);
+
+    if (task === 'creator') {
+        // 返回爬取的 creator 信息
+        res.json({ message: '启动成功', creator: creatorData });
+    } else {
+        res.json({ message: '启动成功' });
+    }
+
+    } catch (error) {
+        res.status(500).json({ error: `启动失败: ${(error as Error).message}` });
+    }
+
+
+}
+
+/**
+ * 手动登录
+ */
+export async function login(req: Request, res: Response): Promise<void> {
+  const { loginType, cookieStr } = req.body;
+
+  if (!loginType) {
+    res.status(400).json({ error: '登录方式 (loginType) 是必填的' });
+    return;
+  }
+
+  try {
+    await XiaoHongshuService.loginXiaoHongShu(loginType, cookieStr);
+    res.json({ message: '登录成功' });
+  } catch (error) {
+    res.status(500).json({ error: `登录失败: ${(error as Error).message}` });
+  }
+}
+
+export async function getAllNotes(req: Request, res: Response): Promise<void> {
+    const { creatorId, crawlInterval, autoLogin, loginType, cookieStr, maxCount } = req.body;
+    logger.info('getAllNotes for xhs');
+
+    if (!creatorId) {
+      res.status(400).json({ error: '用户 ID (creatorId) 是必填的' });
+      return;
+    }
+
+    try {
+      const notes = await XiaoHongshuService.getCreatorAndNotes(
+        creatorId,
+        maxCount,
+        autoLogin,
+        loginType,
+        cookieStr
+      );
+      res.json({ data: notes });
+    } catch (error) {
+      res.status(500).json({ error: `获取用户帖子失败: ${(error as Error).message}` });
+    }
+  }

packages/server/src/crawlers/mediaPlatforms/weibo/login.ts

@@ -39,7 +39,7 @@ export class WeiboLogin implements BaseLogin {
 
     private async checkLoginState(noLoggedInSession?: string): Promise<boolean> {
         const currentCookie = await this.browserContext.cookies();
-        const [cookieDict] = convertCookies(currentCookie);
+        const [cookieStr, cookieDict]: [string, { [key: string]: string }] = convertCookies(currentCookie);
 
         if (cookieDict['SSOLoginState']) {
             return true;
@@ -85,7 +85,7 @@ export class WeiboLogin implements BaseLogin {
         logger.info('[WeiboLogin.login_by_qrcode] Waiting for scan code login...');
 
         const currentCookie = await this.browserContext.cookies();
-        const [cookieDict] = convertCookies(currentCookie);
+        const [coonkieStr, cookieDict] = convertCookies(currentCookie);
         const noLoggedInSession = cookieDict['WBPSESS'];
 
         try {
@@ -122,4 +122,29 @@ export class WeiboLogin implements BaseLogin {
         await this.browserContext.addCookies(cookiesToAdd);
         logger.info('[WeiboLogin.login_by_cookies] Cookies set successfully.');
     }
+    // public async loginByCookies(): Promise<void> {
+    //     logger.info('[WeiboLogin.login_by_cookies] Begin login weibo by cookie ...');
+
+    //     if (!this.cookieStr || this.cookieStr.trim().length === 0) {
+    //         // 如果 cookieStr 为空，则不设置 Cookie，直接返回
+    //         logger.info('[WeiboLogin.login_by_cookies] No cookies provided, skipping cookie login to test guest mode.');
+    //         return;
+    //     }
+
+    //     // 转换 Cookie 字符串为字典
+    //     const cookieDict = convertStrCookieToDict(this.cookieStr);
+    //     const cookiesToAdd = Object.entries(cookieDict).map(([key, value]) => {
+    //         return {
+    //             name: key,
+    //             value: value,
+    //             domain: '.weibo.cn',
+    //             path: '/',
+    //         };
+    //     });
+
+    //     // 将 Cookie 添加到浏览器上下文
+    //     await this.browserContext.addCookies(cookiesToAdd);
+    //     logger.info('[WeiboLogin.login_by_cookies] Cookies set successfully.');
+    // }
+
 }