diff --git a/Utilities/DOCLING_SERVICES/README.md b/Utilities/DOCLING_SERVICES/README.md new file mode 100644 index 00000000..ec551aaa --- /dev/null +++ b/Utilities/DOCLING_SERVICES/README.md @@ -0,0 +1,120 @@ +# DOCLING_SERVICES + +## Overview + +DOCLING_SERVICES is a FastAPI-based application designed to process PDF files and convert them into Markdown format. It leverages the Docling library for efficient document conversion and provides a simple API for seamless integration. + +## Features + +- **PDF to Markdown Conversion:** Easily convert PDF documents to Markdown. +- **Concurrent Processing:** Handles multiple PDF processing tasks concurrently. +- **Configurable Settings:** Customize host, port, and other parameters via `config.json` or environment variables. +- **Logging:** Comprehensive logging for monitoring and debugging. + +## Setup + +### Prerequisites + +- Python 3.8 or higher +- Git + +**Configure Environment (Optional)** + + Customize settings by editing `config.json` or setting environment variables. Default configurations are provided in `config.json`. + +## Running the Service + +### Using the Setup Script + +Execute the setup and run the service: + +1. **Clone the Repository** + ```bash + git clone + cd DOCLING_SERVICES + ``` + +2. **Install Dependencies** + ```bash + ./setup.sh + ``` + +### Manually Starting the Service + +Ensure the virtual environment is activated, then run: +```bash +python -m app.main +``` + +The service will start on the host and port specified in `config.json` (default is `0.0.0.0:8000`). + +## Usage + +### API Endpoint + +- **POST** `/process-pdf-markdown/` + + Upload a PDF file to convert it into Markdown format. + +### Example `curl` Command + +```bash +1. Using File Path (Already on Server) +curl -X POST "http://localhost:8000/process-pdf-markdown/" \ + -H "Content-Type: multipart/form-data" \ + -F "file_path=/tmp/already_uploaded/document.pdf" + +2. Using Form Data (File Upload) +curl -X POST "http://localhost:8000/process-pdf-markdown/" \ + -H "Content-Type: multipart/form-data" \ + -F "file=@/path/to/your/document.pdf" + +3. For doing health Check of the Docling Service +curl http://localhost:8000/health +``` + +### Response Format + +Upon successful processing, the API returns a JSON response: + +```json +{ + "status": "success", + "chunks": [ + { + "chunkText": "Markdown content of page 1", + "filename": "document.pdf", + "page_number": 1 + }, + { + "chunkText": "Markdown content of page 2", + "filename": "document.pdf", + "page_number": 2 + } + // ... more pages + ] +} +``` + +### Error Handling + +If an error occurs during processing, the API responds with an appropriate HTTP status code and error message. + + + +## Logging + +Logs are stored in the `logs/` directory by default. You can monitor the `markdown_service.log` file for detailed insights into the application's operations. + +## Configuration + +Settings can be adjusted in the `config.json` file or via environment variables. Key configurations include: + +- `HOST`: Server host (default: `0.0.0.0`) +- `MARKDOWN_SERVICE_PORT`: Server port (default: `8000`) +- `PDF_THREAD_POOL_SIZE`: Number of worker threads for PDF processing (default: `3`) +- `LOGS_DIR`: Directory for log files (default: `logs`) +- `MARKDOWN_LOG_FILE`: Log file name (default: `markdown_service.log`) + +## License +Kore.ai diff --git a/Utilities/DOCLING_SERVICES/app/__init__.py b/Utilities/DOCLING_SERVICES/app/__init__.py new file mode 100644 index 00000000..14a538bf --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/__init__.py @@ -0,0 +1 @@ +# This file can be empty \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/app/config.py b/Utilities/DOCLING_SERVICES/app/config.py new file mode 100644 index 00000000..970babe7 --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/config.py @@ -0,0 +1,33 @@ +import os +import json +from pathlib import Path +from dotenv import load_dotenv + +# Get the base directory of the service +BASE_DIR = Path(__file__).resolve().parent.parent +ENV_FILE = BASE_DIR / 'config' / '.env' +load_dotenv(ENV_FILE) + +# Try to load configuration from JSON file if it exists +config_data = {} +config_file_path = BASE_DIR / 'config.json' +if config_file_path.exists(): + with open(config_file_path, 'r') as config_file: + config_data = json.load(config_file) + +# Configuration values with environment variable fallbacks +HOST = os.getenv('HOST', config_data.get('HOST', '0.0.0.0')) +MARKDOWN_SERVICE_PORT = int(os.getenv('MARKDOWN_SERVICE_PORT', config_data.get('MARKDOWN_SERVICE_PORT', 8000))) +PDF_THREAD_POOL_SIZE = int(os.getenv('PDF_THREAD_POOL_SIZE', config_data.get('PDF_THREAD_POOL_SIZE', 3))) + +# Logging configuration +LOGS_DIR = os.path.join(BASE_DIR, 'logs') # Relative to service directory +MARKDOWN_LOG_FILE = os.path.join(LOGS_DIR, 'markdown_service.log') + +# Ensure logs directory exists +os.makedirs(LOGS_DIR, exist_ok=True) + +# Processing configuration +MAX_RETRIES = int(os.getenv('MAX_RETRIES', config_data.get('MAX_RETRIES', 3))) +RETRY_DELAY = int(os.getenv('RETRY_DELAY', config_data.get('RETRY_DELAY', 5000))) +REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', config_data.get('REQUEST_TIMEOUT', 300000))) \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/app/main.py b/Utilities/DOCLING_SERVICES/app/main.py new file mode 100644 index 00000000..f5476197 --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/main.py @@ -0,0 +1,35 @@ +from fastapi import FastAPI +import uvicorn +from app.routes.pdf_processing import router as pdf_router +from app.utils.logger import setup_logger +import app.config as config + +# Initialize FastAPI app +app = FastAPI( + title="PDF Markdown Service", + description="Service for converting PDF files to markdown format", + version="1.0.0" +) + +# Setup logger +logger = setup_logger('markdown-service') + +# Include routes +app.include_router(pdf_router) + +@app.on_event("startup") +async def startup_event(): + logger.info(f"Starting PDF Markdown Service on {config.HOST}:{config.MARKDOWN_SERVICE_PORT}") + logger.info(f"Thread pool size: {config.PDF_THREAD_POOL_SIZE}") + +if __name__ == "__main__": + try: + uvicorn.run( + app, + host=config.HOST, + port=config.MARKDOWN_SERVICE_PORT, + log_config=None # Use our custom logging config + ) + except Exception as e: + logger.error(f"Failed to start server: {str(e)}") + raise \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/app/models/processing.py b/Utilities/DOCLING_SERVICES/app/models/processing.py new file mode 100644 index 00000000..40e24d9b --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/models/processing.py @@ -0,0 +1,72 @@ +from docling.datamodel.base_models import InputFormat +from docling.document_converter import DocumentConverter, PdfFormatOption +from docling.datamodel.pipeline_options import PdfPipelineOptions +from docling_core.types.doc import ImageRefMode +from pathlib import Path +import time +import logging +from typing import List, Dict +import app.config as config +import os + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s | %(levelname)s | %(message)s', + handlers=[ + logging.FileHandler(config.MARKDOWN_LOG_FILE), + logging.StreamHandler() + ] +) +logger = logging.getLogger(__name__) + +def ensure_logs_directory(): + """Ensure that the logs directory exists.""" + logs_dir = 'logs' + if not os.path.exists(logs_dir): + os.makedirs(logs_dir) + +def process_uploaded_file_sync(file_path: str) -> List[Dict]: + """Synchronous version of process_uploaded_file""" + # Set up pipeline options + pipeline_options = PdfPipelineOptions() + pipeline_options.do_ocr = False + pipeline_options.do_table_structure = True + pipeline_options.table_structure_options.do_cell_matching = True + pipeline_options.generate_page_images = False + pipeline_options.generate_picture_images = False + + # Create converter instance + doc_converter = DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options) + } + ) + + chunks = [] + input_doc_path = Path(file_path) + + try: + start_time = time.time() + # Convert document + logger.info(f"Started processing Document {file_path}") + conv_result = doc_converter.convert(input_doc_path) + end_time = time.time() - start_time + logger.info(f"Document {file_path} converted in {end_time:.2f} seconds.") + + # Process each page + for page_no, page in conv_result.document.pages.items(): + page_content = conv_result.document.export_to_markdown( + image_mode=ImageRefMode.REFERENCED, + page_no=page_no + ) + chunk_obj = { + 'chunkText': page_content, + 'filename': str(input_doc_path.name), + 'page_number': page_no + } + chunks.append(chunk_obj) + + return chunks + except Exception as e: + logger.error(f"Error processing document {file_path}: {str(e)}") + raise Exception(f"Error processing document: {str(e)}") \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/app/routes/pdf_processing.py b/Utilities/DOCLING_SERVICES/app/routes/pdf_processing.py new file mode 100644 index 00000000..5d381eca --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/routes/pdf_processing.py @@ -0,0 +1,111 @@ +from fastapi import APIRouter, UploadFile, HTTPException, Form +from fastapi.responses import JSONResponse +import time +import logging +from app.services.file_processing import save_and_process_file, process_existing_file +from datetime import datetime +from typing import Optional +import os + +router = APIRouter() + +logger = logging.getLogger(__name__) + +@router.get("/health-check") +async def health_check(): + """Simple health check endpoint to verify the service is running.""" + return JSONResponse( + content={ + "status": "success", + "message": "Service is healthy", + "timestamp": datetime.now().isoformat(), + "service": "pdf-processing-service" + }, + status_code=200 + ) + +@router.post("/process-pdf-markdown/") +async def process_pdf_markdown( + file: Optional[UploadFile] = None, + file_path: Optional[str] = Form(None) +) -> JSONResponse: + # Check if at least one input method is provided + if not file and not file_path: + raise HTTPException( + status_code=400, + detail={ + "status": "error", + "message": "Either file upload or file path must be provided", + "timestamp": datetime.now().isoformat() + } + ) + + start_time = time.time() + + try: + # Process based on the input method + if file: + # Existing file upload flow + if not file.filename.lower().endswith('.pdf'): + raise HTTPException( + status_code=400, + detail={ + "status": "error", + "message": "File must be a PDF", + "timestamp": datetime.now().isoformat() + } + ) + + filename = file.filename + logger.info(f"[MARKDOWN] Starting processing for uploaded file: {filename}") + logger.info(f"[MARKDOWN] Start time: {datetime.fromtimestamp(start_time).isoformat()}") + chunks = await save_and_process_file(file) + else: + # New file path flow + if not file_path.lower().endswith('.pdf'): + raise HTTPException( + status_code=400, + detail={ + "status": "error", + "message": "File must be a PDF", + "timestamp": datetime.now().isoformat() + } + ) + + filename = os.path.basename(file_path) + logger.info(f"[MARKDOWN] Starting processing for file path: {filename}") + logger.info(f"[MARKDOWN] Start time: {datetime.fromtimestamp(start_time).isoformat()}") + chunks = await process_existing_file(file_path) + + processing_time = time.time() - start_time + logger.info(f"[MARKDOWN] Completed processing for file: {filename}") + logger.info(f"[MARKDOWN] Processing time: {processing_time:.2f} seconds") + logger.info(f"[MARKDOWN] End time: {datetime.now().isoformat()}") + + return JSONResponse( + content={ + "status": "success", + "message": f"Successfully processed file: {filename}", + "processing_time_seconds": round(processing_time, 2), + "timestamp": datetime.now().isoformat(), + "chunks": chunks, + "metadata": { + "filename": filename, + "file_type": "pdf" + } + }, + status_code=200 + ) + + except Exception as e: + filename = file.filename if file else os.path.basename(file_path) + logger.error(f"[MARKDOWN] Error processing {filename}: {str(e)}") + raise HTTPException( + status_code=500, + detail={ + "status": "error", + "message": f"Error processing file: {str(e)}", + "timestamp": datetime.now().isoformat(), + "filename": filename + } + ) \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/app/services/file_processing.py b/Utilities/DOCLING_SERVICES/app/services/file_processing.py new file mode 100644 index 00000000..d95ec289 --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/services/file_processing.py @@ -0,0 +1,99 @@ +import os +import tempfile +import asyncio +from fastapi import UploadFile +from concurrent.futures import ProcessPoolExecutor +from typing import List, Dict +from app.models.processing import process_uploaded_file_sync, ensure_logs_directory +import app.config as config +import shutil +import uuid +import logging + +# Set up logging +logger = logging.getLogger(__name__) + +# Create a process pool for concurrent CPU-bound processing +process_pool = ProcessPoolExecutor(max_workers=config.PDF_THREAD_POOL_SIZE) + +# Create a single temporary directory for all file processing +TEMP_DIR = tempfile.mkdtemp(prefix="pdf_processing_") + +# Ensure the temporary directory exists +def ensure_temp_directory(): + """Ensure that the temporary directory exists.""" + try: + if not os.path.exists(TEMP_DIR): + os.makedirs(TEMP_DIR) + logger.info(f"Created temporary directory: {TEMP_DIR}") + return True + except Exception as e: + logger.error(f"Failed to create temporary directory: {str(e)}") + return False + +# Call this at startup +ensure_temp_directory() + +async def save_and_process_file(file: UploadFile) -> List[Dict]: + """Save the uploaded file and process it.""" + # Generate a unique filename to avoid collisions + unique_filename = f"{uuid.uuid4()}_{file.filename}" + temp_file_path = os.path.join(TEMP_DIR, unique_filename) + + logger.info(f"Processing uploaded file: {file.filename}") + logger.info(f"Temporary file path: {temp_file_path}") + + try: + # Save the file + try: + save_uploaded_file(file, temp_file_path) + logger.info(f"File saved successfully to {temp_file_path}") + except Exception as e: + logger.error(f"Failed to save file {file.filename}: {str(e)}") + raise Exception(f"Failed to save file: {str(e)}") + + # Process the file + try: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(process_pool, process_uploaded_file_sync, temp_file_path) + logger.info(f"Successfully processed file: {file.filename}") + return result + except Exception as e: + logger.error(f"Error during file processing for {file.filename}: {str(e)}") + raise Exception(f"Error during file processing: {str(e)}") + finally: + # Clean up the file after processing + try: + if os.path.exists(temp_file_path): + os.remove(temp_file_path) + logger.info(f"Cleaned up temporary file: {temp_file_path}") + except Exception as e: + logger.warning(f"Failed to clean up temporary file {temp_file_path}: {str(e)}") + +async def process_existing_file(file_path: str) -> List[Dict]: + """Process an existing file at the given path.""" + logger.info(f"Processing existing file at path: {file_path}") + + # Verify file exists + if not os.path.exists(file_path): + error_msg = f"File not found at path: {file_path}" + logger.error(error_msg) + raise FileNotFoundError(error_msg) + + try: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(process_pool, process_uploaded_file_sync, file_path) + logger.info(f"Successfully processed existing file: {file_path}") + return result + except Exception as e: + logger.error(f"Error processing existing file {file_path}: {str(e)}") + raise Exception(f"Error processing existing file: {str(e)}") + +def save_uploaded_file(file: UploadFile, destination: str) -> None: + """Save the uploaded file to the specified destination.""" + try: + with open(destination, "wb") as buffer: + shutil.copyfileobj(file.file, buffer) + except Exception as e: + logger.error(f"Error saving file to {destination}: {str(e)}") + raise Exception(f"Error saving file: {str(e)}") \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/app/utils/ensure_logs_dir.py b/Utilities/DOCLING_SERVICES/app/utils/ensure_logs_dir.py new file mode 100644 index 00000000..76439f1a --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/utils/ensure_logs_dir.py @@ -0,0 +1,7 @@ +import os + +def ensure_logs_directory(): + """Ensure that the logs directory exists.""" + logs_dir = 'logs' + if not os.path.exists(logs_dir): + os.makedirs(logs_dir) \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/app/utils/logger.py b/Utilities/DOCLING_SERVICES/app/utils/logger.py new file mode 100644 index 00000000..bda5e656 --- /dev/null +++ b/Utilities/DOCLING_SERVICES/app/utils/logger.py @@ -0,0 +1,25 @@ +import logging +import os +from app.config import LOGS_DIR, MARKDOWN_LOG_FILE + +def setup_logger(name): + """Set up logger with file and console handlers.""" + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + + # Create formatters and handlers + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + # File handler + file_handler = logging.FileHandler(MARKDOWN_LOG_FILE) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # Console handler + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + return logger \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/config.json b/Utilities/DOCLING_SERVICES/config.json new file mode 100644 index 00000000..0a137384 --- /dev/null +++ b/Utilities/DOCLING_SERVICES/config.json @@ -0,0 +1,11 @@ +{ + "HOST": "0.0.0.0", + "MARKDOWN_SERVICE_PORT": 8000, + "PDF_THREAD_POOL_SIZE": 3, + "PDF_PROCESS_POOL_SIZE": 3, + "MAX_RETRIES": 5, + "RETRY_DELAY": 5000, + "REQUEST_TIMEOUT": 300000, + "LOGS_DIR": "logs", + "MARKDOWN_LOG_FILE": "markdown.log" +} \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/logs/markdown_service.log b/Utilities/DOCLING_SERVICES/logs/markdown_service.log new file mode 100644 index 00000000..194239d2 --- /dev/null +++ b/Utilities/DOCLING_SERVICES/logs/markdown_service.log @@ -0,0 +1,174 @@ +2025-03-03 10:24:44,855 | INFO | Started server process [681250] +2025-03-03 10:24:44,855 | INFO | Waiting for application startup. +2025-03-03 10:24:44,855 - markdown-service - INFO - Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-03 10:24:44,855 | INFO | Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-03 10:24:44,855 - markdown-service - INFO - Thread pool size: 3 +2025-03-03 10:24:44,855 | INFO | Thread pool size: 3 +2025-03-03 10:24:44,856 | INFO | Application startup complete. +2025-03-03 10:24:44,856 | INFO | Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2025-03-03 10:29:26,270 | INFO | [MARKDOWN] Starting processing for file: MLA.pdf +2025-03-03 10:29:26,270 | INFO | [MARKDOWN] Start time: 2025-03-03T10:29:26.270215 +2025-03-03 10:29:26,312 | INFO | Started processing Document /tmp/tmpxyckcm3c/MLA.pdf +2025-03-03 10:29:26,330 | INFO | Going to convert document batch... +2025-03-03 10:29:26,332 | INFO | Accelerator device: 'cpu' +2025-03-03 10:29:26,679 | INFO | Accelerator device: 'cpu' +2025-03-03 10:29:28,141 | INFO | Processing document MLA.pdf +2025-03-03 10:29:37,995 | INFO | Finished converting document MLA.pdf in 11.68 sec. +2025-03-03 10:29:37,995 | INFO | Document /tmp/tmpxyckcm3c/MLA.pdf converted in 11.68 seconds. +2025-03-03 10:29:37,998 | INFO | [MARKDOWN] Completed processing for file: MLA.pdf +2025-03-03 10:29:37,998 | INFO | [MARKDOWN] Processing time: 11.73 seconds +2025-03-03 10:29:37,998 | INFO | [MARKDOWN] End time: 2025-03-03T10:29:37.998824 +2025-03-03 10:29:37,999 | INFO | 127.0.0.1:58944 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-03 10:33:28,569 | INFO | [MARKDOWN] Starting processing for file: MLA.pdf +2025-03-03 10:33:28,569 | INFO | [MARKDOWN] Start time: 2025-03-03T10:33:28.569167 +2025-03-03 10:33:28,575 | INFO | Started processing Document /tmp/tmpl0iiv_mg/MLA.pdf +2025-03-03 10:33:28,594 | INFO | Going to convert document batch... +2025-03-03 10:33:28,595 | INFO | Accelerator device: 'cpu' +2025-03-03 10:33:28,958 | INFO | Accelerator device: 'cpu' +2025-03-03 10:33:30,241 | INFO | Processing document MLA.pdf +2025-03-03 10:33:40,560 | INFO | Finished converting document MLA.pdf in 11.98 sec. +2025-03-03 10:33:40,560 | INFO | Document /tmp/tmpl0iiv_mg/MLA.pdf converted in 11.98 seconds. +2025-03-03 10:33:40,562 | INFO | [MARKDOWN] Completed processing for file: MLA.pdf +2025-03-03 10:33:40,563 | INFO | [MARKDOWN] Processing time: 11.99 seconds +2025-03-03 10:33:40,563 | INFO | [MARKDOWN] End time: 2025-03-03T10:33:40.563114 +2025-03-03 10:33:40,563 | INFO | 127.0.0.1:41482 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-03 10:35:08,930 | INFO | [MARKDOWN] Starting processing for file: ocindex_profile_canada_2023.pdf +2025-03-03 10:35:08,930 | INFO | [MARKDOWN] Start time: 2025-03-03T10:35:08.930530 +2025-03-03 10:35:08,936 | INFO | Started processing Document /tmp/tmp871xi9wf/ocindex_profile_canada_2023.pdf +2025-03-03 10:35:08,954 | INFO | Going to convert document batch... +2025-03-03 10:35:08,955 | INFO | Accelerator device: 'cpu' +2025-03-03 10:35:09,291 | INFO | Accelerator device: 'cpu' +2025-03-03 10:35:09,661 | INFO | Processing document ocindex_profile_canada_2023.pdf +2025-03-03 10:35:16,526 | INFO | Finished converting document ocindex_profile_canada_2023.pdf in 7.59 sec. +2025-03-03 10:35:16,527 | INFO | Document /tmp/tmp871xi9wf/ocindex_profile_canada_2023.pdf converted in 7.59 seconds. +2025-03-03 10:35:16,540 | INFO | [MARKDOWN] Completed processing for file: ocindex_profile_canada_2023.pdf +2025-03-03 10:35:16,540 | INFO | [MARKDOWN] Processing time: 7.61 seconds +2025-03-03 10:35:16,540 | INFO | [MARKDOWN] End time: 2025-03-03T10:35:16.540739 +2025-03-03 10:35:16,541 | INFO | 127.0.0.1:40456 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-11 07:40:07,752 | INFO | Started server process [2643] +2025-03-11 07:40:07,753 | INFO | Waiting for application startup. +2025-03-11 07:40:07,753 - markdown-service - INFO - Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-11 07:40:07,753 | INFO | Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-11 07:40:07,753 - markdown-service - INFO - Thread pool size: 3 +2025-03-11 07:40:07,753 | INFO | Thread pool size: 3 +2025-03-11 07:40:07,753 | INFO | Application startup complete. +2025-03-11 07:40:07,753 | INFO | Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2025-03-11 07:42:55,372 | INFO | 127.0.0.1:52752 - "POST /process-pdf-markdown/ HTTP/1.1" 400 +2025-03-11 07:44:56,138 | INFO | 127.0.0.1:46418 - "POST /process-pdf-markdown/ HTTP/1.1" 400 +2025-03-11 07:46:23,594 | ERROR | [MARKDOWN] Error processing ocindex_profile: 400: File must be a PDF +2025-03-11 07:46:23,595 | INFO | 127.0.0.1:40764 - "POST /process-pdf-markdown/ HTTP/1.1" 500 +2025-03-11 07:46:38,611 | INFO | [MARKDOWN] Starting processing for file path: ocindex_profile_france.pdf +2025-03-11 07:46:38,611 | INFO | [MARKDOWN] Start time: 2025-03-11T07:46:38.611798 +2025-03-11 07:46:38,640 | INFO | Started processing Document ./ocindex_profile_france.pdf +2025-03-11 07:46:38,641 | ERROR | Error processing document ./ocindex_profile_france.pdf: [Errno 2] No such file or directory: 'ocindex_profile_france.pdf' +2025-03-11 07:46:38,644 | ERROR | [MARKDOWN] Error processing ocindex_profile_france.pdf: Error processing document: [Errno 2] No such file or directory: 'ocindex_profile_france.pdf' +2025-03-11 07:46:38,645 | INFO | 127.0.0.1:33406 - "POST /process-pdf-markdown/ HTTP/1.1" 500 +2025-03-11 07:46:55,290 | INFO | [MARKDOWN] Starting processing for file path: ocindex_profile_france_2023.pdf +2025-03-11 07:46:55,291 | INFO | [MARKDOWN] Start time: 2025-03-11T07:46:55.290931 +2025-03-11 07:46:55,295 | INFO | Started processing Document ./ocindex_profile_france_2023.pdf +2025-03-11 07:46:55,326 | INFO | Going to convert document batch... +2025-03-11 07:46:55,330 | INFO | Accelerator device: 'cpu' +2025-03-11 07:46:55,988 | INFO | Accelerator device: 'cpu' +2025-03-11 07:46:56,685 | INFO | Processing document ocindex_profile_france_2023.pdf +2025-03-11 07:47:03,032 | INFO | Finished converting document ocindex_profile_france_2023.pdf in 7.74 sec. +2025-03-11 07:47:03,032 | INFO | Document ./ocindex_profile_france_2023.pdf converted in 7.74 seconds. +2025-03-11 07:47:03,045 | INFO | [MARKDOWN] Completed processing for file: ocindex_profile_france_2023.pdf +2025-03-11 07:47:03,045 | INFO | [MARKDOWN] Processing time: 7.75 seconds +2025-03-11 07:47:03,045 | INFO | [MARKDOWN] End time: 2025-03-11T07:47:03.045464 +2025-03-11 07:47:03,045 | INFO | 127.0.0.1:37970 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-11 11:24:53,752 | INFO | Started server process [7177] +2025-03-11 11:24:53,752 | INFO | Waiting for application startup. +2025-03-11 11:24:53,752 - markdown-service - INFO - Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-11 11:24:53,752 | INFO | Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-11 11:24:53,752 - markdown-service - INFO - Thread pool size: 3 +2025-03-11 11:24:53,752 | INFO | Thread pool size: 3 +2025-03-11 11:24:53,752 | INFO | Application startup complete. +2025-03-11 11:24:53,753 | INFO | Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2025-03-11 11:25:10,570 | INFO | [MARKDOWN] Starting processing for file path: ocindex_profile_france_2023.pdf +2025-03-11 11:25:10,570 | INFO | [MARKDOWN] Start time: 2025-03-11T11:25:10.570356 +2025-03-11 11:25:10,570 | INFO | Processing existing file at path: ./ocindex_profile_france_2023.pdf +2025-03-11 11:25:10,599 | INFO | Started processing Document ./ocindex_profile_france_2023.pdf +2025-03-11 11:25:10,616 | INFO | Going to convert document batch... +2025-03-11 11:25:10,618 | INFO | Accelerator device: 'cpu' +2025-03-11 11:25:11,006 | INFO | Accelerator device: 'cpu' +2025-03-11 11:25:11,366 | INFO | Processing document ocindex_profile_france_2023.pdf +2025-03-11 11:25:17,223 | INFO | Finished converting document ocindex_profile_france_2023.pdf in 6.62 sec. +2025-03-11 11:25:17,223 | INFO | Document ./ocindex_profile_france_2023.pdf converted in 6.62 seconds. +2025-03-11 11:25:17,236 | INFO | Successfully processed existing file: ./ocindex_profile_france_2023.pdf +2025-03-11 11:25:17,237 | INFO | [MARKDOWN] Completed processing for file: ocindex_profile_france_2023.pdf +2025-03-11 11:25:17,237 | INFO | [MARKDOWN] Processing time: 6.67 seconds +2025-03-11 11:25:17,237 | INFO | [MARKDOWN] End time: 2025-03-11T11:25:17.237325 +2025-03-11 11:25:17,238 | INFO | 127.0.0.1:43608 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-11 13:09:26,913 | INFO | [MARKDOWN] Starting processing for file path: ocindex_profile_france_2023.pdf +2025-03-11 13:09:26,913 | INFO | [MARKDOWN] Start time: 2025-03-11T13:09:26.913516 +2025-03-11 13:09:26,913 | INFO | Processing existing file at path: ./ocindex_profile_france_2023.pdf +2025-03-11 13:09:26,918 | INFO | Started processing Document ./ocindex_profile_france_2023.pdf +2025-03-11 13:09:26,935 | INFO | Going to convert document batch... +2025-03-11 13:09:26,937 | INFO | Accelerator device: 'cpu' +2025-03-11 13:09:27,329 | INFO | Accelerator device: 'cpu' +2025-03-11 13:09:27,679 | INFO | Processing document ocindex_profile_france_2023.pdf +2025-03-11 13:09:33,533 | INFO | Finished converting document ocindex_profile_france_2023.pdf in 6.61 sec. +2025-03-11 13:09:33,533 | INFO | Document ./ocindex_profile_france_2023.pdf converted in 6.62 seconds. +2025-03-11 13:09:33,546 | INFO | Successfully processed existing file: ./ocindex_profile_france_2023.pdf +2025-03-11 13:09:33,546 | INFO | [MARKDOWN] Completed processing for file: ocindex_profile_france_2023.pdf +2025-03-11 13:09:33,546 | INFO | [MARKDOWN] Processing time: 6.63 seconds +2025-03-11 13:09:33,546 | INFO | [MARKDOWN] End time: 2025-03-11T13:09:33.546706 +2025-03-11 13:09:33,547 | INFO | 127.0.0.1:34022 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-11 13:10:18,183 | INFO | Shutting down +2025-03-11 13:10:18,283 | INFO | Waiting for application shutdown. +2025-03-11 13:10:18,284 | INFO | Application shutdown complete. +2025-03-11 13:10:18,284 | INFO | Finished server process [7177] +2025-03-11 13:10:28,230 | INFO | Started server process [8251] +2025-03-11 13:10:28,230 | INFO | Waiting for application startup. +2025-03-11 13:10:28,230 - markdown-service - INFO - Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-11 13:10:28,230 | INFO | Starting PDF Markdown Service on 0.0.0.0:8000 +2025-03-11 13:10:28,231 - markdown-service - INFO - Thread pool size: 3 +2025-03-11 13:10:28,231 | INFO | Thread pool size: 3 +2025-03-11 13:10:28,231 | INFO | Application startup complete. +2025-03-11 13:10:28,231 | INFO | Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2025-03-11 13:10:35,347 | INFO | [MARKDOWN] Starting processing for file path: ocindex_profile_france_2023.pdf +2025-03-11 13:10:35,347 | INFO | [MARKDOWN] Start time: 2025-03-11T13:10:35.347004 +2025-03-11 13:10:35,347 | INFO | Processing existing file at path: ./ocindex_profile_france_2023.pdf +2025-03-11 13:10:35,379 | INFO | Started processing Document ./ocindex_profile_france_2023.pdf +2025-03-11 13:10:35,396 | INFO | Going to convert document batch... +2025-03-11 13:10:35,398 | INFO | Accelerator device: 'cpu' +2025-03-11 13:10:35,715 | INFO | Accelerator device: 'cpu' +2025-03-11 13:10:36,063 | INFO | Processing document ocindex_profile_france_2023.pdf +2025-03-11 13:10:41,887 | INFO | Finished converting document ocindex_profile_france_2023.pdf in 6.51 sec. +2025-03-11 13:10:41,887 | INFO | Document ./ocindex_profile_france_2023.pdf converted in 6.51 seconds. +2025-03-11 13:10:41,900 | INFO | Successfully processed existing file: ./ocindex_profile_france_2023.pdf +2025-03-11 13:10:41,900 | INFO | [MARKDOWN] Completed processing for file: ocindex_profile_france_2023.pdf +2025-03-11 13:10:41,900 | INFO | [MARKDOWN] Processing time: 6.55 seconds +2025-03-11 13:10:41,901 | INFO | [MARKDOWN] End time: 2025-03-11T13:10:41.900998 +2025-03-11 13:10:41,901 | INFO | 127.0.0.1:57406 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-11 13:10:54,626 | INFO | [MARKDOWN] Starting processing for file path: ocindex_profile_france_2023.pdf +2025-03-11 13:10:54,626 | INFO | [MARKDOWN] Start time: 2025-03-11T13:10:54.626124 +2025-03-11 13:10:54,626 | INFO | Processing existing file at path: ./ocindex_profile_france_2023.pdf +2025-03-11 13:10:54,631 | INFO | Started processing Document ./ocindex_profile_france_2023.pdf +2025-03-11 13:10:54,648 | INFO | Going to convert document batch... +2025-03-11 13:10:54,649 | INFO | Accelerator device: 'cpu' +2025-03-11 13:10:54,973 | INFO | Accelerator device: 'cpu' +2025-03-11 13:10:55,330 | INFO | Processing document ocindex_profile_france_2023.pdf +2025-03-11 13:11:01,081 | INFO | Finished converting document ocindex_profile_france_2023.pdf in 6.45 sec. +2025-03-11 13:11:01,081 | INFO | Document ./ocindex_profile_france_2023.pdf converted in 6.45 seconds. +2025-03-11 13:11:01,094 | INFO | Successfully processed existing file: ./ocindex_profile_france_2023.pdf +2025-03-11 13:11:01,094 | INFO | [MARKDOWN] Completed processing for file: ocindex_profile_france_2023.pdf +2025-03-11 13:11:01,094 | INFO | [MARKDOWN] Processing time: 6.47 seconds +2025-03-11 13:11:01,094 | INFO | [MARKDOWN] End time: 2025-03-11T13:11:01.094773 +2025-03-11 13:11:01,095 | INFO | 127.0.0.1:44026 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-11 13:11:01,108 | INFO | [MARKDOWN] Starting processing for file path: ocindex_profile_france_2023.pdf +2025-03-11 13:11:01,108 | INFO | [MARKDOWN] Start time: 2025-03-11T13:11:01.108781 +2025-03-11 13:11:01,109 | INFO | Processing existing file at path: ./ocindex_profile_france_2023.pdf +2025-03-11 13:11:01,113 | INFO | Started processing Document ./ocindex_profile_france_2023.pdf +2025-03-11 13:11:01,130 | INFO | Going to convert document batch... +2025-03-11 13:11:01,131 | INFO | Accelerator device: 'cpu' +2025-03-11 13:11:01,543 | INFO | Accelerator device: 'cpu' +2025-03-11 13:11:01,899 | INFO | Processing document ocindex_profile_france_2023.pdf +2025-03-11 13:11:07,572 | INFO | Finished converting document ocindex_profile_france_2023.pdf in 6.46 sec. +2025-03-11 13:11:07,573 | INFO | Document ./ocindex_profile_france_2023.pdf converted in 6.46 seconds. +2025-03-11 13:11:07,585 | INFO | Successfully processed existing file: ./ocindex_profile_france_2023.pdf +2025-03-11 13:11:07,585 | INFO | [MARKDOWN] Completed processing for file: ocindex_profile_france_2023.pdf +2025-03-11 13:11:07,585 | INFO | [MARKDOWN] Processing time: 6.48 seconds +2025-03-11 13:11:07,585 | INFO | [MARKDOWN] End time: 2025-03-11T13:11:07.585906 +2025-03-11 13:11:07,586 | INFO | 127.0.0.1:54510 - "POST /process-pdf-markdown/ HTTP/1.1" 200 +2025-03-11 13:11:29,304 | INFO | 127.0.0.1:56786 - "GET /health HTTP/1.1" 200 diff --git a/Utilities/DOCLING_SERVICES/requirements.txt b/Utilities/DOCLING_SERVICES/requirements.txt new file mode 100644 index 00000000..3943522f --- /dev/null +++ b/Utilities/DOCLING_SERVICES/requirements.txt @@ -0,0 +1,9 @@ +docling==2.21.0 +docling-core==2.18.0 +docling-ibm-models==3.3.1 +docling-parse==3.3.0 +easyocr==1.7.2 +fastapi==0.115.8 +python-dotenv==1.0.1 +python-multipart==0.0.20 +uvicorn==0.34.0 \ No newline at end of file diff --git a/Utilities/DOCLING_SERVICES/setup.sh b/Utilities/DOCLING_SERVICES/setup.sh new file mode 100644 index 00000000..4bf9bce7 --- /dev/null +++ b/Utilities/DOCLING_SERVICES/setup.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Exit on any error +set -e + +echo "Setting up DOCLING_SERVICES..." + +# Create virtual environment if it doesn't exist +if [ ! -d "venv" ]; then + echo "Creating virtual environment..." + python3 -m venv venv +fi + +# Activate virtual environment +source venv/bin/activate + +# Install requirements +echo "Installing dependencies..." +pip install -r requirements.txt + +# Create necessary directories +echo "Creating required directories..." +mkdir -p logs + + +echo "Setup completed successfully!" +echo "" + + +# Start the Markdown service +echo "Starting Markdown service..." +python -m app.main \ No newline at end of file