Added Docling Service

Utilities/DOCLING_SERVICES/README.md

@@ -0,0 +1,120 @@
+# DOCLING_SERVICES
+
+## Overview
+
+DOCLING_SERVICES is a FastAPI-based application designed to process PDF files and convert them into Markdown format. It leverages the Docling library for efficient document conversion and provides a simple API for seamless integration.
+
+## Features
+
+- **PDF to Markdown Conversion:** Easily convert PDF documents to Markdown.
+- **Concurrent Processing:** Handles multiple PDF processing tasks concurrently.
+- **Configurable Settings:** Customize host, port, and other parameters via `config.json` or environment variables.
+- **Logging:** Comprehensive logging for monitoring and debugging.
+
+## Setup
+
+### Prerequisites
+
+- Python 3.8 or higher
+- Git
+
+**Configure Environment (Optional)**
+
+   Customize settings by editing `config.json` or setting environment variables. Default configurations are provided in `config.json`.
+
+## Running the Service
+
+### Using the Setup Script
+
+Execute the setup and run the service:
+
+1. **Clone the Repository**
+    ```bash
+    git clone <repository-url>
+    cd DOCLING_SERVICES
+    ```
+
+2. **Install Dependencies**
+    ```bash
+    ./setup.sh
+    ```
+
+### Manually Starting the Service
+
+Ensure the virtual environment is activated, then run:
+```bash
+python -m app.main
+```
+
+The service will start on the host and port specified in `config.json` (default is `0.0.0.0:8000`).
+
+## Usage
+
+### API Endpoint
+
+- **POST** `/process-pdf-markdown/`
+
+    Upload a PDF file to convert it into Markdown format.
+
+### Example `curl` Command
+
+```bash
+1. Using File Path (Already on Server)
+curl -X POST "http://localhost:8000/process-pdf-markdown/" \
+  -H "Content-Type: multipart/form-data" \
+  -F "file_path=/tmp/already_uploaded/document.pdf"
+
+2. Using Form Data (File Upload)
+curl -X POST "http://localhost:8000/process-pdf-markdown/" \
+  -H "Content-Type: multipart/form-data" \
+  -F "file=@/path/to/your/document.pdf"
+
+3. For doing health Check of the Docling Service
+curl http://localhost:8000/health
+```
+
+### Response Format
+
+Upon successful processing, the API returns a JSON response:
+
+```json
+{
+  "status": "success",
+  "chunks": [
+    {
+      "chunkText": "Markdown content of page 1",
+      "filename": "document.pdf",
+      "page_number": 1
+    },
+    {
+      "chunkText": "Markdown content of page 2",
+      "filename": "document.pdf",
+      "page_number": 2
+    }
+    // ... more pages
+  ]
+}
+```
+
+### Error Handling
+
+If an error occurs during processing, the API responds with an appropriate HTTP status code and error message.
+
+
+
+## Logging
+
+Logs are stored in the `logs/` directory by default. You can monitor the `markdown_service.log` file for detailed insights into the application's operations.
+
+## Configuration
+
+Settings can be adjusted in the `config.json` file or via environment variables. Key configurations include:
+
+- `HOST`: Server host (default: `0.0.0.0`)
+- `MARKDOWN_SERVICE_PORT`: Server port (default: `8000`)
+- `PDF_THREAD_POOL_SIZE`: Number of worker threads for PDF processing (default: `3`)
+- `LOGS_DIR`: Directory for log files (default: `logs`)
+- `MARKDOWN_LOG_FILE`: Log file name (default: `markdown_service.log`)
+
+## License
+Kore.ai

Utilities/DOCLING_SERVICES/app/__init__.py

@@ -0,0 +1 @@
+# This file can be empty 

Utilities/DOCLING_SERVICES/app/config.py

@@ -0,0 +1,33 @@
+import os
+import json
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Get the base directory of the service
+BASE_DIR = Path(__file__).resolve().parent.parent
+ENV_FILE = BASE_DIR / 'config' / '.env'
+load_dotenv(ENV_FILE)
+
+# Try to load configuration from JSON file if it exists
+config_data = {}
+config_file_path = BASE_DIR / 'config.json'
+if config_file_path.exists():
+    with open(config_file_path, 'r') as config_file:
+        config_data = json.load(config_file)
+
+# Configuration values with environment variable fallbacks
+HOST = os.getenv('HOST', config_data.get('HOST', '0.0.0.0'))
+MARKDOWN_SERVICE_PORT = int(os.getenv('MARKDOWN_SERVICE_PORT', config_data.get('MARKDOWN_SERVICE_PORT', 8000)))
+PDF_THREAD_POOL_SIZE = int(os.getenv('PDF_THREAD_POOL_SIZE', config_data.get('PDF_THREAD_POOL_SIZE', 3)))
+
+# Logging configuration
+LOGS_DIR = os.path.join(BASE_DIR, 'logs')  # Relative to service directory
+MARKDOWN_LOG_FILE = os.path.join(LOGS_DIR, 'markdown_service.log')
+
+# Ensure logs directory exists
+os.makedirs(LOGS_DIR, exist_ok=True)
+
+# Processing configuration
+MAX_RETRIES = int(os.getenv('MAX_RETRIES', config_data.get('MAX_RETRIES', 3)))
+RETRY_DELAY = int(os.getenv('RETRY_DELAY', config_data.get('RETRY_DELAY', 5000)))
+REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', config_data.get('REQUEST_TIMEOUT', 300000))) 

Utilities/DOCLING_SERVICES/app/main.py

@@ -0,0 +1,35 @@
+from fastapi import FastAPI
+import uvicorn
+from app.routes.pdf_processing import router as pdf_router
+from app.utils.logger import setup_logger
+import app.config as config
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="PDF Markdown Service",
+    description="Service for converting PDF files to markdown format",
+    version="1.0.0"
+)
+
+# Setup logger
+logger = setup_logger('markdown-service')
+
+# Include routes
+app.include_router(pdf_router)
+
+@app.on_event("startup")
+async def startup_event():
+    logger.info(f"Starting PDF Markdown Service on {config.HOST}:{config.MARKDOWN_SERVICE_PORT}")
+    logger.info(f"Thread pool size: {config.PDF_THREAD_POOL_SIZE}")
+
+if __name__ == "__main__":
+    try:
+        uvicorn.run(
+            app,
+            host=config.HOST,
+            port=config.MARKDOWN_SERVICE_PORT,
+            log_config=None  # Use our custom logging config
+        )
+    except Exception as e:
+        logger.error(f"Failed to start server: {str(e)}")
+        raise 

Utilities/DOCLING_SERVICES/app/models/processing.py

@@ -0,0 +1,72 @@
+from docling.datamodel.base_models import InputFormat
+from docling.document_converter import DocumentConverter, PdfFormatOption
+from docling.datamodel.pipeline_options import PdfPipelineOptions
+from docling_core.types.doc import ImageRefMode
+from pathlib import Path
+import time
+import logging
+from typing import List, Dict
+import app.config as config
+import os
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s | %(levelname)s | %(message)s',
+    handlers=[
+        logging.FileHandler(config.MARKDOWN_LOG_FILE),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+def ensure_logs_directory():
+    """Ensure that the logs directory exists."""
+    logs_dir = 'logs'
+    if not os.path.exists(logs_dir):
+        os.makedirs(logs_dir)
+
+def process_uploaded_file_sync(file_path: str) -> List[Dict]:
+    """Synchronous version of process_uploaded_file"""
+    # Set up pipeline options
+    pipeline_options = PdfPipelineOptions()
+    pipeline_options.do_ocr = False
+    pipeline_options.do_table_structure = True
+    pipeline_options.table_structure_options.do_cell_matching = True
+    pipeline_options.generate_page_images = False
+    pipeline_options.generate_picture_images = False
+
+    # Create converter instance
+    doc_converter = DocumentConverter(
+        format_options={
+            InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
+        }
+    )
+
+    chunks = []
+    input_doc_path = Path(file_path)
+
+    try:
+        start_time = time.time()
+        # Convert document
+        logger.info(f"Started processing Document {file_path}")
+        conv_result = doc_converter.convert(input_doc_path)
+        end_time = time.time() - start_time
+        logger.info(f"Document {file_path} converted in {end_time:.2f} seconds.")
+
+        # Process each page
+        for page_no, page in conv_result.document.pages.items():
+            page_content = conv_result.document.export_to_markdown(
+                image_mode=ImageRefMode.REFERENCED,
+                page_no=page_no
+            )
+            chunk_obj = {
+                'chunkText': page_content,
+                'filename': str(input_doc_path.name),
+                'page_number': page_no
+            }
+            chunks.append(chunk_obj)
+
+        return chunks
+    except Exception as e:
+        logger.error(f"Error processing document {file_path}: {str(e)}")
+        raise Exception(f"Error processing document: {str(e)}") 

Utilities/DOCLING_SERVICES/app/routes/pdf_processing.py

@@ -0,0 +1,111 @@
+from fastapi import APIRouter, UploadFile, HTTPException, Form
+from fastapi.responses import JSONResponse
+import time
+import logging
+from app.services.file_processing import save_and_process_file, process_existing_file
+from datetime import datetime
+from typing import Optional
+import os
+
+router = APIRouter()
+
+logger = logging.getLogger(__name__)
+
+@router.get("/health-check")
+async def health_check():
+    """Simple health check endpoint to verify the service is running."""
+    return JSONResponse(
+        content={
+            "status": "success",
+            "message": "Service is healthy",
+            "timestamp": datetime.now().isoformat(),
+            "service": "pdf-processing-service"
+        },
+        status_code=200
+    )
+
+@router.post("/process-pdf-markdown/")
+async def process_pdf_markdown(
+    file: Optional[UploadFile] = None, 
+    file_path: Optional[str] = Form(None)
+) -> JSONResponse:
+    # Check if at least one input method is provided
+    if not file and not file_path:
+        raise HTTPException(
+            status_code=400, 
+            detail={
+                "status": "error",
+                "message": "Either file upload or file path must be provided",
+                "timestamp": datetime.now().isoformat()
+            }
+        )
+
+    start_time = time.time()
+
+    try:
+        # Process based on the input method
+        if file:
+            # Existing file upload flow
+            if not file.filename.lower().endswith('.pdf'):
+                raise HTTPException(
+                    status_code=400, 
+                    detail={
+                        "status": "error",
+                        "message": "File must be a PDF",
+                        "timestamp": datetime.now().isoformat()
+                    }
+                )
+
+            filename = file.filename
+            logger.info(f"[MARKDOWN] Starting processing for uploaded file: {filename}")
+            logger.info(f"[MARKDOWN] Start time: {datetime.fromtimestamp(start_time).isoformat()}")
+            chunks = await save_and_process_file(file)
+        else:
+            # New file path flow
+            if not file_path.lower().endswith('.pdf'):
+                raise HTTPException(
+                    status_code=400, 
+                    detail={
+                        "status": "error",
+                        "message": "File must be a PDF",
+                        "timestamp": datetime.now().isoformat()
+                    }
+                )
+
+            filename = os.path.basename(file_path)
+            logger.info(f"[MARKDOWN] Starting processing for file path: {filename}")
+            logger.info(f"[MARKDOWN] Start time: {datetime.fromtimestamp(start_time).isoformat()}")
+            chunks = await process_existing_file(file_path)
+
+        processing_time = time.time() - start_time
+        logger.info(f"[MARKDOWN] Completed processing for file: {filename}")
+        logger.info(f"[MARKDOWN] Processing time: {processing_time:.2f} seconds")
+        logger.info(f"[MARKDOWN] End time: {datetime.now().isoformat()}")
+
+        return JSONResponse(
+            content={
+                "status": "success",
+                "message": f"Successfully processed file: {filename}",
+                "processing_time_seconds": round(processing_time, 2),
+                "timestamp": datetime.now().isoformat(),
+                "chunks": chunks,
+                "metadata": {
+                    "filename": filename,
+                    "file_type": "pdf"
+                }
+            }, 
+            status_code=200
+        )
+
+    except Exception as e:
+        filename = file.filename if file else os.path.basename(file_path)
+        logger.error(f"[MARKDOWN] Error processing {filename}: {str(e)}")
+        raise HTTPException(
+            status_code=500, 
+            detail={
+                "status": "error",
+                "message": f"Error processing file: {str(e)}",
+                "timestamp": datetime.now().isoformat(),
+                "filename": filename
+            }
+        )