From 3e4880b8b60641c2478026d4a0d9c5e8794c73f3 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 1 Feb 2026 19:14:17 +0100 Subject: [PATCH 01/10] Initial commit with task details Adding CLAUDE.md with task information for AI processing. This file will be removed when the task is complete. Issue: https://github.com/Payel-git-ol/TgCrawler/issues/1 --- CLAUDE.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..9ab42ef --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,7 @@ +Issue to solve: https://github.com/Payel-git-ol/TgCrawler/issues/1 +Your prepared branch: issue-1-28dc8a7552ee +Your prepared working directory: /tmp/gh-issue-solver-1769969647890 +Your forked repository: konard/Payel-git-ol-TgCrawler +Original repository (upstream): Payel-git-ol/TgCrawler + +Proceed. From 7c491f252eb48b100ffea98b453592bb3fec18af Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 1 Feb 2026 19:17:25 +0100 Subject: [PATCH 02/10] fix: correct DATABASE_URL to use POSTGRES_DB env var and add url to Prisma schema This fixes the Prisma "table does not exist" error by ensuring: 1. DATABASE_URL in docker-compose uses ${POSTGRES_DB} instead of hardcoded 'telegram-scrapper' 2. Prisma schema.prisma includes url = env("DATABASE_URL") in datasource block 3. Removed unused init-db volume mount from postgres service The error occurred because the database was created as 'tgcrawler' (from POSTGRES_DB) but the app was trying to connect to 'telegram-scrapper'. Prisma migrations are already configured to run automatically via Dockerfile CMD. Co-Authored-By: Claude Sonnet 4.5 --- docker-compose.yaml | 3 +-- prisma/schema.prisma | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 94df107..9d7dc46 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,7 +21,7 @@ services: - HOST=0.0.0.0 - TZ=Europe/Moscow - PLAYWRIGHT_EXECUTABLE_PATH=/usr/bin/chromium-browser - - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/telegram-scrapper + - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} - START_ON_BOOT=true - START_ON_BOOT_DELAY_MS=10000 - START_ON_BOOT_ATTEMPTS=5 @@ -45,7 +45,6 @@ services: - "5432:5432" volumes: - ${DATA_DIR}/pg:/var/lib/postgresql - - ./init-db:/docker-entrypoint-initdb.d healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] interval: 5s diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f8a0bef..484a782 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5,6 +5,7 @@ generator client { datasource db { provider = "postgresql" + url = env("DATABASE_URL") } model Task { From 411db87ec43b4691fca9c996cdc58228c2f9e68f Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 1 Feb 2026 19:19:48 +0100 Subject: [PATCH 03/10] Revert "Initial commit with task details" This reverts commit 3e4880b8b60641c2478026d4a0d9c5e8794c73f3. --- CLAUDE.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 9ab42ef..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,7 +0,0 @@ -Issue to solve: https://github.com/Payel-git-ol/TgCrawler/issues/1 -Your prepared branch: issue-1-28dc8a7552ee -Your prepared working directory: /tmp/gh-issue-solver-1769969647890 -Your forked repository: konard/Payel-git-ol-TgCrawler -Original repository (upstream): Payel-git-ol/TgCrawler - -Proceed. From df39a6edde62545405fad385804e08d65530c4a5 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 1 Feb 2026 19:27:07 +0100 Subject: [PATCH 04/10] fix: remove url from schema.prisma for Prisma 7 compatibility Prisma 7 no longer supports the `url` property in schema.prisma datasource block. The DATABASE_URL is already correctly configured in prisma.config.ts (for migrations/CLI) and via the PrismaPg adapter in TaskService (for runtime). Co-Authored-By: Claude Opus 4.5 --- prisma/schema.prisma | 1 - 1 file changed, 1 deletion(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 484a782..f8a0bef 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5,7 +5,6 @@ generator client { datasource db { provider = "postgresql" - url = env("DATABASE_URL") } model Task { From bc21f81ba5292d77c041ef0fb090b770a41d4d03 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 1 Feb 2026 19:47:30 +0100 Subject: [PATCH 05/10] feat: add date parameter to crawl endpoint and CLI command Allow crawling since a specific date instead of only the last 7 days. - POST /api/crawl accepts optional {date: "YYYY-MM-DD"} in request body - npm run crawl supports --date 2025-01-15 CLI argument - Scraper.scrape() accepts optional sinceDate parameter - Default behavior unchanged (last 7 days when no date specified) Co-Authored-By: Claude Opus 4.5 --- src/crawler/crawl-only.ts | 23 +++++++++++++++++------ src/index.ts | 22 ++++++++++++++++++---- src/services/srcaper/scraper.ts | 13 ++++++------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/crawler/crawl-only.ts b/src/crawler/crawl-only.ts index 2f7590b..e20e869 100644 --- a/src/crawler/crawl-only.ts +++ b/src/crawler/crawl-only.ts @@ -6,15 +6,26 @@ import { ServiceFactory } from "../services/factory"; import { launchBrowser } from "../launchBrowser"; async function crawl(): Promise { + // Parse --date argument (e.g. --date 2025-01-15) + const dateArgIndex = process.argv.indexOf('--date'); + let sinceDate: Date | undefined; + if (dateArgIndex !== -1 && process.argv[dateArgIndex + 1]) { + const parsed = new Date(process.argv[dateArgIndex + 1]); + if (isNaN(parsed.getTime())) { + Logger.error(`Invalid date: ${process.argv[dateArgIndex + 1]}. Use ISO format (e.g. 2025-01-15)`); + process.exit(1); + } + sinceDate = parsed; + } + const browser = await launchBrowser(); const scraper = ServiceFactory.createScraper(); const storage = new DataStorage(CONFIG.DATA_DIR); - const oneWeekAgo = new Date(); - oneWeekAgo.setDate(oneWeekAgo.getDate() - CONFIG.MAX_POST_AGE_DAYS); + const cutoffDate = sinceDate || new Date(Date.now() - CONFIG.MAX_POST_AGE_DAYS * 24 * 60 * 60 * 1000); try { - Logger.info("Starting Telegram Crawl (Last Week Only)"); - Logger.info(`Looking for posts newer than: ${oneWeekAgo.toLocaleDateString()}`); + Logger.info(`Starting Telegram Crawl (since ${cutoffDate.toISOString().split('T')[0]})`); + Logger.info(`Looking for posts newer than: ${cutoffDate.toLocaleDateString()}`); const existingIds = await storage.getExistingIds(); const existingContent = await storage.getExistingContent(); @@ -32,7 +43,7 @@ async function crawl(): Promise { const page = await browser.newPage(); try { - const allPosts = await scraper.scrape(page, url); + const allPosts = await scraper.scrape(page, url, sinceDate); if (allPosts.length === 0) { Logger.warn(` No posts found from ${url}`); @@ -44,7 +55,7 @@ async function crawl(): Promise { // Проверяем время публикации if (p.timestamp) { const postDate = new Date(p.timestamp); - if (postDate < oneWeekAgo) { + if (postDate < cutoffDate) { totalFilteredByAge++; return false; } diff --git a/src/index.ts b/src/index.ts index cf1c416..4c16085 100644 --- a/src/index.ts +++ b/src/index.ts @@ -479,7 +479,7 @@ app.get("/", (c) => { "GET /api/file/:filename": "Get posts from specific file", "GET /api/jobs/:id": "Get post by ID", "GET /api/jobs/filter/type/:type": "Filter by work type", - "POST /api/crawl": "Start crawler", + "POST /api/crawl": "Start crawler (optional body: {date: 'YYYY-MM-DD'} to crawl since a specific date)", "GET /api/stats": "Get statistics", }, "Task Management": { @@ -596,7 +596,7 @@ app.get("/api/jobs/filter/type/:type", async (c) => { }); // Extracted crawl runner so it can be used by API and on-start hooks -async function runCrawlAndSave(): Promise<{ success: boolean; message?: string; posts?: JobPost[]; error?: any }> { +async function runCrawlAndSave(sinceDate?: Date): Promise<{ success: boolean; message?: string; posts?: JobPost[]; error?: any }> { const browser = await launchBrowser(); const scraper = ServiceFactory.createScraper(); const allCollected: JobPost[] = []; @@ -622,7 +622,7 @@ async function runCrawlAndSave(): Promise<{ success: boolean; message?: string; const page = await browser.newPage(); try { - const allPosts = await scraper.scrape(page, url); + const allPosts = await scraper.scrape(page, url, sinceDate); const newPosts = allPosts.filter((p) => { const contentHash = `${p.title}|${p.description}`.toLowerCase().trim(); @@ -697,7 +697,21 @@ async function runCrawlAndSave(): Promise<{ success: boolean; message?: string; } app.post("/api/crawl", async (c) => { - const result = await runCrawlAndSave(); + let sinceDate: Date | undefined; + try { + const body = await c.req.json().catch(() => ({})); + if (body.date) { + const parsed = new Date(body.date); + if (isNaN(parsed.getTime())) { + return c.json({ success: false, error: "Invalid date format. Use ISO 8601 (e.g. 2025-01-15) or any Date-parseable string." }, 400); + } + sinceDate = parsed; + } + } catch { + // No body or invalid JSON — proceed with defaults + } + + const result = await runCrawlAndSave(sinceDate); if (result.success) { return c.json({ success: true, message: result.message, posts: result.posts }); } else { diff --git a/src/services/srcaper/scraper.ts b/src/services/srcaper/scraper.ts index bad6c1b..c9c4511 100644 --- a/src/services/srcaper/scraper.ts +++ b/src/services/srcaper/scraper.ts @@ -7,7 +7,7 @@ import { Logger } from "../../log/logger"; export class Scraper { constructor(private extractor: PostExtractor, private htmlParser: HtmlParser) {} - async scrape(page: Page, url: string): Promise { + async scrape(page: Page, url: string, sinceDate?: Date): Promise { Logger.info(`Starting page load: ${url}`); await page.goto(url, { waitUntil: "networkidle" }); @@ -22,8 +22,7 @@ export class Scraper { const allPosts = []; let lastPostId: string | null = null; let foundOldPosts = false; - const oneWeekAgo = new Date(); - oneWeekAgo.setDate(oneWeekAgo.getDate() - 7); + const cutoffDate = sinceDate || new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); for (let i = 0; i < CONFIG.MAX_CRAWL_ITERATIONS && !foundOldPosts; i++) { const posts = await this.extractor.extractFromPage(page, url); @@ -41,11 +40,11 @@ export class Scraper { if (!post.timestamp) return true; const postDate = new Date(post.timestamp); - return postDate >= oneWeekAgo; + return postDate >= cutoffDate; }); if (recentPosts.length === 0) { - Logger.info("Reached posts older than 1 week, stopping"); + Logger.info("Reached posts older than cutoff date, stopping"); foundOldPosts = true; break; } @@ -53,7 +52,7 @@ export class Scraper { allPosts.push(...recentPosts); if (recentPosts.length < posts.length) { - Logger.info("Reached posts older than 1 week, stopping"); + Logger.info("Reached posts older than cutoff date, stopping"); foundOldPosts = true; break; } @@ -69,7 +68,7 @@ export class Scraper { await this.htmlParser.scrollPage(page, 2); } - Logger.success(`Collected ${allPosts.length} posts from the last week`); + Logger.success(`Collected ${allPosts.length} posts since ${cutoffDate.toISOString().split('T')[0]}`); return allPosts; } } \ No newline at end of file From a2f9b2df886bf1e2efdec1747dfa523d489f6210 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 1 Feb 2026 20:02:53 +0100 Subject: [PATCH 06/10] fix: fix crawler not finding posts due to broken selectors and strict validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: The crawler returned "No new posts found" despite posts existing because of multiple compounding bugs: 1. ContentValidator.isJobPost() required BOTH job keywords AND job emojis (👔/💼/📌). Many valid job posts don't use these specific emojis. Changed to accept posts with keywords OR emojis. 2. HtmlParser.extractPostId() used selector `a[href*='/s/']` to find post links, but Telegram post URLs are `t.me/channel/ID` (no `/s/`). This caused all posts to get random IDs (`post_${Date.now()}`), breaking deduplication. Fixed to use the `data-post` attribute on message elements. 3. HtmlParser MESSAGE_TIME selector was `time.datetime` (looking for class "datetime") but the actual `