Add attachment summarizer service

attachment-summarizer-service/.env.example

@@ -0,0 +1,14 @@
+DATABASE_URL="postgresql://postgres:postgres@postgres:5432/attachments?schema=public"
+AWS_REGION="us-east-1"
+SQS_QUEUE_URL="https://sqs.us-east-1.amazonaws.com/123456789012/attachment-events"
+GOOGLE_CLOUD_PROJECT="warpspeed-open"
+GCS_BUCKET="mail-attachments"
+OLLAMA_BASE_URL="http://ollama:11434"
+OLLAMA_MODEL="llama3.1:8b"
+WORKER_CONCURRENCY="4"
+SQS_WAIT_TIME_SECONDS="20"
+SQS_VISIBILITY_TIMEOUT_SECONDS="120"
+MAX_ATTACHMENT_BYTES="26214400"
+OCR_IMAGES="true"
+LOG_LEVEL="info"
+

attachment-summarizer-service/Dockerfile

@@ -0,0 +1,22 @@
+FROM node:20-bookworm-slim AS deps
+WORKDIR /app
+COPY package.json package-lock.json* ./
+RUN npm install
+
+FROM deps AS build
+COPY tsconfig.json ./
+COPY prisma ./prisma
+COPY src ./src
+RUN npx prisma generate
+RUN npm run build
+
+FROM node:20-bookworm-slim AS runtime
+WORKDIR /app
+ENV NODE_ENV=production
+COPY package.json package-lock.json* ./
+RUN npm install --omit=dev
+COPY prisma ./prisma
+COPY --from=build /app/node_modules/.prisma ./node_modules/.prisma
+COPY --from=build /app/dist ./dist
+CMD ["node", "dist/index.js"]
+

attachment-summarizer-service/README.md

@@ -0,0 +1,92 @@
+# Attachment Summarizer Service
+
+This implementation is a production-oriented Node.js/TypeScript worker for the
+warpSpeed OPEN Attachment Summarizer bounty.
+
+It consumes attachment events from AWS SQS, downloads files from Google Cloud
+Storage, extracts text from common attachment types, summarizes the content with
+a locally hosted Ollama model, and persists job state with Prisma.
+
+## Features
+
+- AWS SQS long-polling worker with visibility timeout configuration
+- Google Cloud Storage attachment download
+- Prisma-backed idempotent job tracking
+- Supported extraction paths:
+  - plain text
+  - HTML
+  - CSV / TSV
+  - PDF
+  - Word `.docx`
+  - spreadsheets `.xlsx` / `.xls`
+  - image OCR via `tesseract.js`
+- Local LLM summarization through Ollama
+- Docker and Docker Compose setup
+- Unit tests for parsing, validation, and summarization prompt shaping
+- Structured logging and retry-friendly error states
+
+## Event Format
+
+SQS messages should contain JSON like:
+
+```json
+{
+  "eventId": "evt_123",
+  "messageId": "email_456",
+  "bucket": "mail-attachments",
+  "objectName": "inbox/user/document.pdf",
+  "filename": "document.pdf",
+  "mimeType": "application/pdf",
+  "sizeBytes": 123456
+}
+```
+
+`eventId`, `bucket`, `objectName`, and `filename` are required.
+
+## Local Setup
+
+```bash
+cp .env.example .env
+docker compose up -d postgres ollama
+npm install
+npm run prisma:generate
+npm run build
+npm test
+```
+
+Pull an Ollama model before running the worker:
+
+```bash
+docker compose exec ollama ollama pull llama3.1:8b
+```
+
+Run database migrations:
+
+```bash
+npx prisma migrate dev --name init
+```
+
+Start the worker:
+
+```bash
+npm run dev
+```
+
+## Production Notes
+
+- Use Workload Identity or `GOOGLE_APPLICATION_CREDENTIALS` for GCS auth.
+- Configure AWS credentials through the normal AWS SDK provider chain.
+- Keep `MAX_ATTACHMENT_BYTES` conservative to avoid worker memory spikes.
+- Set `OCR_IMAGES=false` if OCR latency is too high for the deployment tier.
+- Keep Ollama on the same private network as the worker.
+
+## Bounty Mapping
+
+- Consumes SQS events: `src/queue/sqsConsumer.ts`
+- Downloads GCS objects: `src/storage/gcsClient.ts`
+- Extracts supported file types: `src/extractors/index.ts`
+- Summarizes with local LLM: `src/llm/ollamaClient.ts`
+- Persists state with Prisma: `src/persistence/attachmentRepository.ts`
+- Worker orchestration: `src/worker.ts`
+- Docker setup: `Dockerfile`, `docker-compose.yml`
+

attachment-summarizer-service/SUBMISSION.md

@@ -0,0 +1,28 @@
+# Submission Notes
+
+## Summary
+
+Implemented a Node.js/TypeScript attachment summarizer worker that processes SQS
+events, downloads attachments from GCS, extracts content, generates factual
+summaries with Ollama, and stores processing state with Prisma.
+
+## Scope Covered
+
+- SQS consumer with long polling
+- GCS download integration
+- Text extraction for text, HTML, CSV/TSV, PDF, DOCX, XLS/XLSX, and images
+- Ollama local LLM client
+- Prisma schema and repository layer
+- Docker/Docker Compose setup
+- Error handling and structured logs
+- Tests and static validation script
+
+## Payment Preference
+
+Preferred payout, if accepted by the maintainer:
+
+- Solana: `J8hHY978whDthFVrRrR7ypgyQ4jH6zpqfzzwVbWuERuv`
+- Ethereum: `0xaeD3eE0637Fc7C7e5F6a920A8e000706D1cc7e6C`
+
+PayPal can be provided by the human owner if platform payout requires it.
+

attachment-summarizer-service/docker-compose.yml

@@ -0,0 +1,31 @@
+services:
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_DB: attachments
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+
+  ollama:
+    image: ollama/ollama:latest
+    ports:
+      - "11434:11434"
+    volumes:
+      - ollama-data:/root/.ollama
+
+  worker:
+    build: .
+    depends_on:
+      - postgres
+      - ollama
+    env_file:
+      - .env
+
+volumes:
+  postgres-data:
+  ollama-data:
+

attachment-summarizer-service/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "warpspeed-attachment-summarizer",
+  "version": "1.0.0",
+  "description": "SQS + GCS attachment summarizer service using Prisma and a local Ollama LLM.",
+  "type": "module",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "dev": "tsx watch src/index.ts",
+    "start": "node dist/index.js",
+    "test": "vitest run",
+    "prisma:generate": "prisma generate",
+    "prisma:migrate": "prisma migrate deploy",
+    "check:static": "node scripts/static-check.mjs"
+  },
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.600.0",
+    "@google-cloud/storage": "^7.11.0",
+    "@prisma/client": "^5.16.0",
+    "cheerio": "^1.0.0-rc.12",
+    "dotenv": "^16.4.5",
+    "mammoth": "^1.8.0",
+    "pdf-parse": "^1.1.1",
+    "pino": "^9.3.2",
+    "tesseract.js": "^5.1.0",
+    "xlsx": "^0.18.5",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/node": "^20.14.10",
+    "prisma": "^5.16.0",
+    "tsx": "^4.16.2",
+    "typescript": "^5.5.3",
+    "vitest": "^1.6.0"
+  }
+}
+

attachment-summarizer-service/prisma/schema.prisma

@@ -0,0 +1,40 @@
+generator client {
+  provider = "prisma-client-js"
+}
+
+datasource db {
+  provider = "postgresql"
+  url      = env("DATABASE_URL")
+}
+
+model AttachmentJob {
+  id              String    @id @default(cuid())
+  eventId         String    @unique
+  messageId       String?
+  bucket          String
+  objectName      String
+  gcsGeneration   String?
+  filename        String
+  mimeType        String?
+  sizeBytes       Int?
+  status          JobStatus @default(PENDING)
+  extractedText   String?
+  summary         String?
+  summaryModel    String?
+  errorMessage    String?
+  attempts        Int       @default(0)
+  createdAt       DateTime  @default(now())
+  updatedAt       DateTime  @updatedAt
+  completedAt     DateTime?
+
+  @@index([status])
+  @@index([bucket, objectName])
+}
+
+enum JobStatus {
+  PENDING
+  PROCESSING
+  COMPLETED
+  FAILED
+}
+

attachment-summarizer-service/scripts/static-check.mjs

@@ -0,0 +1,56 @@
+import fs from "node:fs";
+import path from "node:path";
+
+const root = process.cwd();
+const requiredFiles = [
+  "package.json",
+  "tsconfig.json",
+  "Dockerfile",
+  "docker-compose.yml",
+  ".env.example",
+  "prisma/schema.prisma",
+  "src/index.ts",
+  "src/worker.ts",
+  "src/queue/sqsConsumer.ts",
+  "src/storage/gcsClient.ts",
+  "src/extractors/index.ts",
+  "src/llm/ollamaClient.ts",
+  "src/persistence/attachmentRepository.ts",
+  "test/event-validation.test.ts",
+  "test/extractors.test.ts",
+  "test/ollama-prompt.test.ts",
+  "README.md",
+  "SUBMISSION.md",
+];
+
+const requiredTokens = [
+  ["package.json", "@aws-sdk/client-sqs"],
+  ["package.json", "@google-cloud/storage"],
+  ["package.json", "@prisma/client"],
+  ["src/config.ts", "OLLAMA_BASE_URL"],
+  ["prisma/schema.prisma", "model AttachmentJob"],
+  ["src/queue/sqsConsumer.ts", "ReceiveMessageCommand"],
+  ["src/storage/gcsClient.ts", "file.download"],
+  ["src/extractors/index.ts", "pdfParse"],
+  ["src/extractors/index.ts", "mammoth"],
+  ["src/extractors/index.ts", "Tesseract"],
+  ["src/llm/ollamaClient.ts", "/api/generate"],
+  ["Dockerfile", "npm run build"],
+  ["docker-compose.yml", "ollama/ollama"],
+];
+
+for (const file of requiredFiles) {
+  const target = path.join(root, file);
+  if (!fs.existsSync(target)) {
+    throw new Error(`Missing required file: ${file}`);
+  }
+}
+
+for (const [file, token] of requiredTokens) {
+  const content = fs.readFileSync(path.join(root, file), "utf8");
+  if (!content.includes(token)) {
+    throw new Error(`Missing token "${token}" in ${file}`);
+  }
+}
+
+console.log("static-check ok");

attachment-summarizer-service/src/config.ts

@@ -0,0 +1,21 @@
+import "dotenv/config";
+import { z } from "zod";
+
+const envSchema = z.object({
+  DATABASE_URL: z.string().min(1),
+  AWS_REGION: z.string().min(1).default("us-east-1"),
+  SQS_QUEUE_URL: z.string().url(),
+  GOOGLE_CLOUD_PROJECT: z.string().optional(),
+  GCS_BUCKET: z.string().min(1),
+  OLLAMA_BASE_URL: z.string().url().default("http://localhost:11434"),
+  OLLAMA_MODEL: z.string().min(1).default("llama3.1:8b"),
+  WORKER_CONCURRENCY: z.coerce.number().int().positive().max(20).default(4),
+  SQS_WAIT_TIME_SECONDS: z.coerce.number().int().min(1).max(20).default(20),
+  SQS_VISIBILITY_TIMEOUT_SECONDS: z.coerce.number().int().min(30).max(43200).default(120),
+  MAX_ATTACHMENT_BYTES: z.coerce.number().int().positive().default(25 * 1024 * 1024),
+  OCR_IMAGES: z.coerce.boolean().default(true),
+  LOG_LEVEL: z.string().default("info"),
+});
+
+export const config = envSchema.parse(process.env);
+