Fix Prisma container error and crawler not finding posts

Dockerfile

@@ -40,5 +40,10 @@ ENV TZ=UTC
 
 EXPOSE 3000
 
-# Автоматически применить миграции Prisma перед запуском приложения
-CMD ["sh", "-c", "npx prisma migrate deploy && node dist/index.js"]
+# Copy startup script that runs prisma generate + migrate deploy before starting the app.
+# NOTE: prisma migrate deploy MUST run at startup (not during build) because it needs
+# a running database. The database is only available at runtime via docker-compose.
+COPY docker-entrypoint.sh /app/docker-entrypoint.sh
+RUN chmod +x /app/docker-entrypoint.sh
+
+CMD ["/app/docker-entrypoint.sh"]

docker-compose.yaml

@@ -21,7 +21,7 @@ services:
       - HOST=0.0.0.0
       - TZ=Europe/Moscow
       - PLAYWRIGHT_EXECUTABLE_PATH=/usr/bin/chromium-browser
-      - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/telegram-scrapper
+      - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
       - START_ON_BOOT=true
       - START_ON_BOOT_DELAY_MS=10000
       - START_ON_BOOT_ATTEMPTS=5
@@ -45,7 +45,6 @@ services:
       - "5432:5432"
     volumes:
       - ${DATA_DIR}/pg:/var/lib/postgresql
-      - ./init-db:/docker-entrypoint-initdb.d
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
       interval: 5s

docker-entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -e
+
+echo "=== Starting TgCrawler ==="
+
+# Generate Prisma client (ensures client matches current schema)
+echo "Running prisma generate..."
+npx prisma generate || echo "WARNING: prisma generate failed, using build-time generated client"
+
+# Apply database migrations (requires DATABASE_URL and running database)
+echo "Running prisma migrate deploy..."
+npx prisma migrate deploy || echo "WARNING: prisma migrate deploy failed, app will create tables via SQL fallback"
+
+# Start the application
+echo "Starting application..."
+exec node dist/index.js

prisma/migrations/20260125085858_init/migration.sql

@@ -14,3 +14,6 @@ CREATE TABLE "Task" (
 
     CONSTRAINT "Task_pkey" PRIMARY KEY ("id")
 );
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Task_id_post_key" ON "Task"("id_post");

src/crawler/crawl-only.ts

@@ -4,22 +4,59 @@ import { CONFIG } from "../config/config";
 import { Logger } from "../log/logger";
 import { ServiceFactory } from "../services/factory";
 import { launchBrowser } from "../launchBrowser";
+import { TaskService } from "../services/database/task";
+import { config } from 'dotenv';
+
+config();
 
 async function crawl(): Promise<void> {
+  // Parse --date argument (e.g. --date 2025-01-15)
+  const dateArgIndex = process.argv.indexOf('--date');
+  let sinceDate: Date | undefined;
+  if (dateArgIndex !== -1 && process.argv[dateArgIndex + 1]) {
+    const parsed = new Date(process.argv[dateArgIndex + 1]);
+    if (isNaN(parsed.getTime())) {
+      Logger.error(`Invalid date: ${process.argv[dateArgIndex + 1]}. Use ISO format (e.g. 2025-01-15)`);
+      process.exit(1);
+    }
+    sinceDate = parsed;
+  }
+
   const browser = await launchBrowser();
   const scraper = ServiceFactory.createScraper();
   const storage = new DataStorage(CONFIG.DATA_DIR);
-  const oneWeekAgo = new Date();
-  oneWeekAgo.setDate(oneWeekAgo.getDate() - CONFIG.MAX_POST_AGE_DAYS);
+
+  // Initialize database service if DATABASE_URL is available
+  let taskService: TaskService | null = null;
+  try {
+    taskService = new TaskService();
+    await taskService.ensureTablesExist();
+    const dbCount = await taskService.countTasks();
+    Logger.info(`Database connection OK. Current tasks in DB: ${dbCount}`);
+  } catch (dbError) {
+    Logger.warn(`Database not available, saving to files only: ${dbError}`);
+    taskService = null;
+  }
+
+  const cutoffDate = sinceDate || new Date(Date.now() - CONFIG.MAX_POST_AGE_DAYS * 24 * 60 * 60 * 1000);
 
   try {
-    Logger.info("Starting Telegram Crawl (Last Week Only)");
-    Logger.info(`Looking for posts newer than: ${oneWeekAgo.toLocaleDateString()}`);
+    Logger.info(`Starting Telegram Crawl (since ${cutoffDate.toISOString().split('T')[0]})`);
+    Logger.info(`Looking for posts newer than: ${cutoffDate.toLocaleDateString()}`);
 
-    const existingIds = await storage.getExistingIds();
-    const existingContent = await storage.getExistingContent();
+    // Use DB for dedup if available, otherwise fall back to file storage
+    let existingIds: Set<string>;
+    let existingContent: Set<string>;
+
+    if (taskService) {
+      existingIds = await taskService.getExistingPostIds();
+      existingContent = await taskService.getExistingContentHashes();
+    } else {
+      existingIds = await storage.getExistingIds();
+      existingContent = await storage.getExistingContent();
+    }
     Logger.info(
-      `Database: ${existingIds.size} posts, ${existingContent.size} content hashes`
+      `Existing: ${existingIds.size} posts, ${existingContent.size} content hashes`
     );
 
     let totalSaved = 0;
@@ -32,8 +69,8 @@ async function crawl(): Promise<void> {
       const page = await browser.newPage();
 
       try {
-        const allPosts = await scraper.scrape(page, url);
-        
+        const allPosts = await scraper.scrape(page, url, sinceDate);
+
         if (allPosts.length === 0) {
           Logger.warn(`  No posts found from ${url}`);
           continue;
@@ -44,14 +81,14 @@ async function crawl(): Promise<void> {
           // Проверяем время публикации
           if (p.timestamp) {
             const postDate = new Date(p.timestamp);
-            if (postDate < oneWeekAgo) {
+            if (postDate < cutoffDate) {
               totalFilteredByAge++;
               return false;
             }
           }
-          
+
           // Проверяем дубликаты
-          const contentHash = `${p.title}|${p.description}`;
+          const contentHash = `${p.title}|${p.description}`.toLowerCase().trim();
           return !existingIds.has(p.id) && !existingContent.has(contentHash);
         });
 
@@ -60,20 +97,45 @@ async function crawl(): Promise<void> {
           continue;
         }
 
-        const result = await storage.saveNewJobs(newPosts);
+        // Save to database if available
+        let dbSavedCount = 0;
+        if (taskService) {
+          try {
+            const tasksToSave = newPosts.map((post) => ({
+              id_post: post.id,
+              title: post.title,
+              description: post.description,
+              workType: post.workType,
+              payment: post.payment,
+              deadline: post.deadline,
+              url: post.url,
+              channelUrl: post.channelUrl || "",
+              scrapedAt: post.scrapedAt,
+              timestamp: post.timestamp || post.scrapedAt,
+            }));
+            const dbResult = await taskService.createManyTasks(tasksToSave);
+            dbSavedCount = dbResult.count;
+            Logger.success(`  Saved to DB: ${dbSavedCount}`);
+          } catch (dbError) {
+            Logger.error(`  Database save failed: ${dbError}`);
+          }
+        }
 
-        Logger.success(`  Saved: ${result.saved.length}`);
-        Logger.info(`  Duplicates: ${result.skipped}`);
+        // Also save to files
+        const fileResult = await storage.saveNewJobs(newPosts);
+
+        Logger.success(`  Saved to files: ${fileResult.saved.length}`);
+        Logger.info(`  Duplicates: ${fileResult.skipped}`);
         Logger.info(`  Filtered by age: ${totalFilteredByAge}`);
 
-        totalSaved += result.saved.length;
-        totalDuplicates += result.skipped;
-        allSavedPosts.push(...result.saved);
+        totalSaved += Math.max(dbSavedCount, fileResult.saved.length);
+        totalDuplicates += fileResult.skipped;
+        allSavedPosts.push(...newPosts);
 
         // Update existing IDs and content for next channel
-        result.saved.forEach((p) => {
+        newPosts.forEach((p) => {
           existingIds.add(p.id);
-          existingContent.add(`${p.title}|${p.description}`);
+          existingContent.add(`${p.title}|${p.description}`.toLowerCase().trim());
         });
       } finally {
         await page.close();
@@ -98,8 +160,11 @@ async function crawl(): Promise<void> {
   } catch (error) {
     Logger.error("Crawl failed", error);
   } finally {
+    if (taskService) {
+      await taskService.disconnect();
+    }
     await browser.close();
   }
 }
 
-crawl();
+crawl();