implement new api endpoint (#129)

Author: aliamerj

rag-engine/src/layers/chunking_embedding/chunk_document.py

@@ -51,6 +51,8 @@ def chunk_document(
 
     # ---- FINAL CLEANUP ----
     chunks = _deduplicate_chunks(chunks)
+    if len(chunks) == 0:
+        raise ValueError("No text found in your pdf!, make sure it is not image")
 
     return chunks
 

rag-engine/src/process/__init__.py …ers/data_extractor/extractor/__init__.pyrag-engine/src/process/__init__.py renamed to rag-engine/src/layers/data_extractor/extractor/__init__.py rag-engine/src/process/__init__.py renamed to rag-engine/src/layers/data_extractor/extractor/__init__.py

@@ -17,13 +17,12 @@
 # ===============================
 # PUBLIC ENTRY
 # ===============================
-def pdf(pdf_bytes: bytes) -> tuple[list[Page], dict]:
+def extract_data(pdf_bytes: bytes) -> tuple[list[Page], dict]:
     pages_output: list[Page] = []
     metadata = {}
 
     try:
         with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf_doc:
-            metadata["_document_id"] = str(uuid.uuid4())
             metadata["_file_type"] = "pdf"
             metadata["_page_count"] = len(pdf_doc.pages)
             metadata["_file_metadata"] = pdf_doc.metadata

…e/src/layers/data_extractor/extractor.py …c/layers/data_extractor/extractor/pdf.pyrag-engine/src/layers/data_extractor/extractor.py renamed to rag-engine/src/layers/data_extractor/extractor/pdf.py rag-engine/src/layers/data_extractor/extractor.py renamed to rag-engine/src/layers/data_extractor/extractor/pdf.py

@@ -1,13 +1,14 @@
 from dotenv import load_dotenv
 from fastapi import FastAPI
-from src.process.controller import router as process
+from src.store.routers import store_upload_router, store_url_router
 from .logging import configure_logging, LogLevels
 from pathlib import Path
 
 
-env_path = Path(__file__).parent / '.env'
+env_path = Path(__file__).parent.parent / '.env'
 
 load_dotenv(dotenv_path=env_path)
 configure_logging(LogLevels.info)
 app = FastAPI()
-app.include_router(process)
+app.include_router(store_upload_router)
+app.include_router(store_url_router)