From 5d7bd8727263cd649f6ae4eef75204eee0cfb6ec Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Sun, 15 Feb 2026 05:02:56 +0200 Subject: [PATCH 1/8] init project directory --- .github/.workflows/deploy-develop.yml | 46 --- .vscode/settings.json | 3 - README.md | 151 --------- docker/.gitignore | 3 - docker/README.md | 183 ----------- docker/docker-compose.yml | 250 -------------- docker/env/.env.example.app | 49 --- docker/env/.env.example.grafana | 4 - docker/env/.env.example.postgres | 4 - docker/env/.env.example.postgres-exporter | 3 - docker/env/.env.example.rabbitmq | 13 - docker/env/.env.example.redis | 12 - docker/minirag.service | 23 -- docker/minirag/Dockerfile | 35 -- docker/minirag/alembic.example.ini | 117 ------- docker/minirag/entrypoint.sh | 7 - docker/nginx/default.conf | 17 - docker/prometheus/prometheus.yml | 27 -- docker/rabbitmq/rabbitmq.conf | 18 -- src/.env.example | 50 --- src/.gitignore | 163 ---------- src/assets/.gitignore | 2 - src/assets/.gitkeep | 0 .../mini-rag-app.postman_collection.json | 57 ---- src/celery_app.py | 107 ------ src/controllers/BaseController.py | 35 -- src/controllers/DataController.py | 57 ---- src/controllers/NLPController.py | 141 -------- src/controllers/ProcessController.py | 109 ------- src/controllers/ProjectController.py | 22 -- src/controllers/__init__.py | 5 - src/flowerconfig.py | 12 - src/helpers/__init__.py | 0 src/helpers/config.py | 57 ---- src/main.py | 61 ---- src/models/AssetModel.py | 50 --- src/models/BaseDataModel.py | 7 - src/models/ChunkModel.py | 69 ---- src/models/ProjectModel.py | 61 ---- src/models/__init__.py | 3 - src/models/db_schemes/__init__.py | 1 - src/models/db_schemes/minirag/.gitignore | 1 - src/models/db_schemes/minirag/README.md | 21 -- src/models/db_schemes/minirag/__init__.py | 0 .../db_schemes/minirag/alembic.ini.example | 117 ------- src/models/db_schemes/minirag/alembic/README | 1 - src/models/db_schemes/minirag/alembic/env.py | 79 ----- .../db_schemes/minirag/alembic/script.py.mako | 26 -- ...b0_update_celery_task_executions_table_.py | 30 -- ...09b_create_celery_task_executions_table.py | 51 --- .../versions/fee4cd54bd38_initial_commit.py | 76 ----- .../db_schemes/minirag/schemes/__init__.py | 5 - .../db_schemes/minirag/schemes/asset.py | 32 -- .../minirag/schemes/celery_task_execution.py | 34 -- .../db_schemes/minirag/schemes/datachunk.py | 36 --- .../minirag/schemes/minirag_base.py | 2 - .../db_schemes/minirag/schemes/project.py | 18 -- src/models/enums/AssetTypeEnum.py | 6 - src/models/enums/DataBaseEnum.py | 8 - src/models/enums/ProcessingEnum.py | 6 - src/models/enums/ResponseEnums.py | 24 -- src/models/enums/__init__.py | 0 src/requirements.txt | 32 -- src/routes/__init__.py | 0 src/routes/base.py | 23 -- src/routes/data.py | 136 -------- src/routes/nlp.py | 139 -------- src/routes/schemes/__init__.py | 0 src/routes/schemes/data.py | 8 - src/routes/schemes/nlp.py | 9 - src/stores/llm/LLMEnums.py | 23 -- src/stores/llm/LLMInterface.py | 24 -- src/stores/llm/LLMProviderFactory.py | 27 -- src/stores/llm/__init__.py | 0 src/stores/llm/providers/CoHereProvider.py | 101 ------ src/stores/llm/providers/OpenAIProvider.py | 109 ------- src/stores/llm/providers/__init__.py | 2 - src/stores/llm/templates/__init__.py | 0 src/stores/llm/templates/locales/__init__.py | 0 .../llm/templates/locales/ar/__init__.py | 0 src/stores/llm/templates/locales/ar/rag.py | 33 -- .../llm/templates/locales/en/__init__.py | 0 src/stores/llm/templates/locales/en/rag.py | 33 -- src/stores/llm/templates/template_parser.py | 43 --- src/stores/vectordb/VectorDBEnums.py | 25 -- src/stores/vectordb/VectorDBInterface.py | 52 --- .../vectordb/VectorDBProviderFactory.py | 31 -- src/stores/vectordb/__init__.py | 0 .../vectordb/providers/PGVectorProvider.py | 306 ------------------ .../vectordb/providers/QdrantDBProvider.py | 152 --------- src/stores/vectordb/providers/__init__.py | 2 - src/tasks/__init__.py | 0 src/tasks/data_indexing.py | 144 --------- src/tasks/file_processing.py | 262 --------------- src/tasks/maintenance.py | 50 --- src/tasks/process_workflow.py | 54 ---- src/utils/__init__.py | 0 src/utils/idempotency_manager.py | 124 ------- src/utils/metrics.py | 36 --- 99 files changed, 4587 deletions(-) delete mode 100644 .github/.workflows/deploy-develop.yml delete mode 100644 .vscode/settings.json delete mode 100644 README.md delete mode 100644 docker/.gitignore delete mode 100644 docker/README.md delete mode 100644 docker/docker-compose.yml delete mode 100644 docker/env/.env.example.app delete mode 100644 docker/env/.env.example.grafana delete mode 100644 docker/env/.env.example.postgres delete mode 100644 docker/env/.env.example.postgres-exporter delete mode 100644 docker/env/.env.example.rabbitmq delete mode 100644 docker/env/.env.example.redis delete mode 100644 docker/minirag.service delete mode 100644 docker/minirag/Dockerfile delete mode 100644 docker/minirag/alembic.example.ini delete mode 100644 docker/minirag/entrypoint.sh delete mode 100644 docker/nginx/default.conf delete mode 100644 docker/prometheus/prometheus.yml delete mode 100644 docker/rabbitmq/rabbitmq.conf delete mode 100644 src/.env.example delete mode 100644 src/.gitignore delete mode 100644 src/assets/.gitignore delete mode 100644 src/assets/.gitkeep delete mode 100644 src/assets/mini-rag-app.postman_collection.json delete mode 100644 src/celery_app.py delete mode 100644 src/controllers/BaseController.py delete mode 100644 src/controllers/DataController.py delete mode 100644 src/controllers/NLPController.py delete mode 100644 src/controllers/ProcessController.py delete mode 100644 src/controllers/ProjectController.py delete mode 100644 src/controllers/__init__.py delete mode 100644 src/flowerconfig.py delete mode 100644 src/helpers/__init__.py delete mode 100644 src/helpers/config.py delete mode 100644 src/main.py delete mode 100644 src/models/AssetModel.py delete mode 100644 src/models/BaseDataModel.py delete mode 100644 src/models/ChunkModel.py delete mode 100644 src/models/ProjectModel.py delete mode 100644 src/models/__init__.py delete mode 100644 src/models/db_schemes/__init__.py delete mode 100644 src/models/db_schemes/minirag/.gitignore delete mode 100644 src/models/db_schemes/minirag/README.md delete mode 100644 src/models/db_schemes/minirag/__init__.py delete mode 100644 src/models/db_schemes/minirag/alembic.ini.example delete mode 100644 src/models/db_schemes/minirag/alembic/README delete mode 100644 src/models/db_schemes/minirag/alembic/env.py delete mode 100644 src/models/db_schemes/minirag/alembic/script.py.mako delete mode 100644 src/models/db_schemes/minirag/alembic/versions/243ca8b683b0_update_celery_task_executions_table_.py delete mode 100644 src/models/db_schemes/minirag/alembic/versions/b9f9e870b09b_create_celery_task_executions_table.py delete mode 100644 src/models/db_schemes/minirag/alembic/versions/fee4cd54bd38_initial_commit.py delete mode 100644 src/models/db_schemes/minirag/schemes/__init__.py delete mode 100644 src/models/db_schemes/minirag/schemes/asset.py delete mode 100644 src/models/db_schemes/minirag/schemes/celery_task_execution.py delete mode 100644 src/models/db_schemes/minirag/schemes/datachunk.py delete mode 100644 src/models/db_schemes/minirag/schemes/minirag_base.py delete mode 100644 src/models/db_schemes/minirag/schemes/project.py delete mode 100644 src/models/enums/AssetTypeEnum.py delete mode 100644 src/models/enums/DataBaseEnum.py delete mode 100644 src/models/enums/ProcessingEnum.py delete mode 100644 src/models/enums/ResponseEnums.py delete mode 100644 src/models/enums/__init__.py delete mode 100644 src/requirements.txt delete mode 100644 src/routes/__init__.py delete mode 100644 src/routes/base.py delete mode 100644 src/routes/data.py delete mode 100644 src/routes/nlp.py delete mode 100644 src/routes/schemes/__init__.py delete mode 100644 src/routes/schemes/data.py delete mode 100644 src/routes/schemes/nlp.py delete mode 100644 src/stores/llm/LLMEnums.py delete mode 100644 src/stores/llm/LLMInterface.py delete mode 100644 src/stores/llm/LLMProviderFactory.py delete mode 100644 src/stores/llm/__init__.py delete mode 100644 src/stores/llm/providers/CoHereProvider.py delete mode 100644 src/stores/llm/providers/OpenAIProvider.py delete mode 100644 src/stores/llm/providers/__init__.py delete mode 100644 src/stores/llm/templates/__init__.py delete mode 100644 src/stores/llm/templates/locales/__init__.py delete mode 100644 src/stores/llm/templates/locales/ar/__init__.py delete mode 100644 src/stores/llm/templates/locales/ar/rag.py delete mode 100644 src/stores/llm/templates/locales/en/__init__.py delete mode 100644 src/stores/llm/templates/locales/en/rag.py delete mode 100644 src/stores/llm/templates/template_parser.py delete mode 100644 src/stores/vectordb/VectorDBEnums.py delete mode 100644 src/stores/vectordb/VectorDBInterface.py delete mode 100644 src/stores/vectordb/VectorDBProviderFactory.py delete mode 100644 src/stores/vectordb/__init__.py delete mode 100644 src/stores/vectordb/providers/PGVectorProvider.py delete mode 100644 src/stores/vectordb/providers/QdrantDBProvider.py delete mode 100644 src/stores/vectordb/providers/__init__.py delete mode 100644 src/tasks/__init__.py delete mode 100644 src/tasks/data_indexing.py delete mode 100644 src/tasks/file_processing.py delete mode 100644 src/tasks/maintenance.py delete mode 100644 src/tasks/process_workflow.py delete mode 100644 src/utils/__init__.py delete mode 100644 src/utils/idempotency_manager.py delete mode 100644 src/utils/metrics.py diff --git a/.github/.workflows/deploy-develop.yml b/.github/.workflows/deploy-develop.yml deleted file mode 100644 index f71ac8940..000000000 --- a/.github/.workflows/deploy-develop.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Deploy Develop Branch to Server - -on: - push: - branches: - - develop - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4.2.2 - - - name: Deploy via SSH - uses: appleboy/ssh-action@v1.2.2 - with: - host: ${{ secrets.SSH_DEVELOP_HOST_IP }} - username: github_user - key: ${{ secrets.SSH_DEVELOP_PRIVATE_KEY }} - script: | - cd /home/github_user/workspace/mini-rag - git checkout develop - git pull - sudo systemctl restart minirag.service - echo "Waiting..." - sleep 20 - - for i in {1..6}; do - if ss -tuln | grep -q ':80'; then - echo "✅ Port 80 is now active." - break - else - echo "⏳ Port 80 not ready yet. Retrying in 5 seconds..." - sleep 5 - fi - done - - if ! ss -tuln | grep -q ':80'; then - echo "❌ Service failed to start on port 80" - exit 1 - fi - - - diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 99ff45e02..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.languageServer": "Pylance" -} \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index d59c47f8f..000000000 --- a/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# mini-rag - -This is a minimal implementation of the RAG model for question answering. - -## The Course - -This is an educational project where all of the codes where explained (step by step) via a set of `Arabic` youtube videos. Please check the list: - -| # | Title | Link | Codes | -|---|------------------------------------------|------------------------------------------------------------------------------------------------------|----------------------------------------------------| -| 1 | About the Course ماذا ولمـــاذا | [Video](https://www.youtube.com/watch?v=Vv6e2Rb1Q6w&list=PLvLvlVqNQGHCUR2p0b8a0QpVjDUg50wQj) | NA | -| 2 | What will we build ماذا سنبنى في المشروع | [Video](https://www.youtube.com/watch?v=_l5S5CdxE-Q&list=PLvLvlVqNQGHCUR2p0b8a0QpVjDUg50wQj&index=2) | NA | -| 3 | Setup your tools الأدوات الأساسية | [Video](https://www.youtube.com/watch?v=VSFbkFRAT4w&list=PLvLvlVqNQGHCUR2p0b8a0QpVjDUg50wQj&index=3) | NA | -| 4 | Project Architecture | [Video](https://www.youtube.com/watch?v=Ei_nBwBbFUQ&list=PLvLvlVqNQGHCUR2p0b8a0QpVjDUg50wQj&index=4) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-001) | -| 5 | Welcome to FastAPI | [Video](https://www.youtube.com/watch?v=cpOuCdzN_Mo&list=PLvLvlVqNQGHCUR2p0b8a0QpVjDUg50wQj&index=5) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-002) | -| 6 | Nested Routes + Env Values | [Video](https://www.youtube.com/watch?v=CrR2Bz2Y7Hw&list=PLvLvlVqNQGHCUR2p0b8a0QpVjDUg50wQj&index=6) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-003) | -| 7 | Uploading a File | [Video](https://www.youtube.com/watch?v=5alMKCbFqWs&list=PLvLvlVqNQGHCUR2p0b8a0QpVjDUg50wQj&index=7) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-004) | -| 8 | File Processing | [Video](https://www.youtube.com/watch?v=gQgr2iwtSBw) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-005) | -| 9 | Docker - MongoDB - Motor | [Video](https://www.youtube.com/watch?v=2NOKWm0xJAk) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-006) | -| 10 | Mongo Schemes and Models | [Video](https://www.youtube.com/watch?v=zgcnnMJXXV8) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-007) | -| 11 | Mongo Indexing | [Video](https://www.youtube.com/watch?v=iO8FAmUVcjE) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-008) | -| 12 | Data Pipeline Enhancements | [Video](https://www.youtube.com/watch?v=4x1DuezZBDU) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-008) | -| 13 | Checkpoint-1 | [Video](https://www.youtube.com/watch?v=7xIsZkCisPk) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-008) | -| 14 | LLM Factory | [Video](https://www.youtube.com/watch?v=5TKRIFtIQAY) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-008) | -| 15 | Vector DB Factory | [Video](https://www.youtube.com/watch?v=JtS9UkvF_10) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-009) | -| 16 | Semantic Search | [Video](https://www.youtube.com/watch?v=V3swQKokJW8) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-010) | -| 17 | Augmented Answers | [Video](https://www.youtube.com/watch?v=1Wx8BoM5pLU) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-011) | -| 18 | Checkpoint-1 + Fix Issues | [Video](https://youtu.be/6zG4Idxldvg) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-012) | -| 19 | Ollama Local LLM Server | [Video](https://youtu.be/-epZ1hAAtrs) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-012) | -| 20 | From Mongo to Postgres + SQLAlchemy & Alembic | [Video](https://www.youtube.com/watch?v=BVOq7Ek2Up0) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-013) | -| 21 | The way to PgVector | [Video](https://www.youtube.com/watch?v=g99yq5zlYAE) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-014) | -| 22 | App Deployments 1/2 | [Video](https://www.youtube.com/watch?v=7QRPnAbVssg) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-015) | -| 22 | App Deployments 2/2 | [Video](https://www.youtube.com/watch?v=qJ5Hdyc4hDc) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-015) | -| 24 | Celery Workers 1/2 | [Video](https://www.youtube.com/watch?v=pX-iWWT2TJo) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-016) | -| 25 | Celery Workers 2/2 | [Video](https://www.youtube.com/watch?v=SZ5Aznjf8Kc) | [branch](https://github.com/bakrianoo/mini-rag/tree/tut-017) | - - - - -## Requirements - -- Python 3.10 - -#### Install Dependencies - -```bash -sudo apt update -sudo apt install libpq-dev gcc python3-dev -``` - -#### Install Python using MiniConda - -1) Download and install MiniConda from [here](https://docs.anaconda.com/free/miniconda/#quick-command-line-install) -2) Create a new environment using the following command: -```bash -$ conda create -n mini-rag python=3.10 -``` -3) Activate the environment: -```bash -$ conda activate mini-rag -``` - -### (Optional) Setup you command line interface for better readability - -```bash -export PS1="\[\033[01;32m\]\u@\h:\w\n\[\033[00m\]\$ " -``` - -### (Optional) Run Ollama Local LLM Server using Colab + Ngrok - -- Check the [notebook](https://colab.research.google.com/drive/1KNi3-9KtP-k-93T3wRcmRe37mRmGhL9p?usp=sharing) + [Video](https://youtu.be/-epZ1hAAtrs) - -## Installation - -### Install the required packages - -```bash -$ pip install -r requirements.txt -``` - -### Setup the environment variables - -```bash -$ cp .env.example .env -``` - -### Run Alembic Migration - -```bash -$ alembic upgrade head -``` - -Set your environment variables in the `.env` file. Like `OPENAI_API_KEY` value. - -## Run Docker Compose Services - -```bash -$ cd docker -$ cp .env.example .env -``` - -- update `.env` with your credentials - - - -```bash -$ cd docker -$ sudo docker compose up -d -``` - -## Access Services - -- **FastAPI**: http://localhost:8000 -- **Flower Dashboard**: http://localhost:5555 (admin/password from env) -- **Grafana**: http://localhost:3000 -- **Prometheus**: http://localhost:9090 - -## Run the FastAPI server (Development Mode) - -```bash -$ uvicorn main:app --reload --host 0.0.0.0 --port 5000 -``` - -# Celery (Development Mode) - -For development, you can run Celery services manually instead of using Docker: - -To Run the **Celery worker**, you need to run the following command in a separate terminal: - -```bash -$ python -m celery -A celery_app worker --queues=default,file_processing,data_indexing --loglevel=info -``` - -To run the **Beat scheduler**, you can run the following command in a separate terminal: - -```bash -$ python -m celery -A celery_app beat --loglevel=info -``` - -To Run **Flower Dashboard**, you can run the following command in a separate terminal: - -```bash -$ python -m celery -A celery_app flower --conf=flowerconfig.py -``` - - -open your browser and go to `http://localhost:5555` to see the dashboard. - -## POSTMAN Collection - -Download the POSTMAN collection from [/assets/mini-rag-app.postman_collection.json](/assets/mini-rag-app.postman_collection.json) diff --git a/docker/.gitignore b/docker/.gitignore deleted file mode 100644 index 975aa757a..000000000 --- a/docker/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -env/.env* -!env/.env.example.* -minirag/alembic.ini diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index fba41cad5..000000000 --- a/docker/README.md +++ /dev/null @@ -1,183 +0,0 @@ -# Docker Setup for MiniRAG Application - -This directory contains the Docker setup for the MiniRAG application, including all necessary services for development and monitoring. - -## Services - -- **FastAPI Application**: Main application running on Uvicorn -- **Nginx**: Web server for serving the FastAPI application -- **PostgreSQL (pgvector)**: Vector-enabled database for storing embeddings -- **Postgres-Exporter**: Exports PostgreSQL metrics for Prometheus -- **Qdrant**: Vector database for similarity search -- **Prometheus**: Metrics collection -- **Grafana**: Visualization dashboard for metrics -- **Node-Exporter**: System metrics collection - -## Setup Instructions - -### 1. Set up environment files - -Create your environment files from the examples: - -```bash -# Create all required .env files from examples -cd docker/env -cp .env.example.app .env.app -cp .env.example.postgres .env.postgres -cp .env.example.grafana .env.grafana -cp .env.example.postgres-exporter .env.postgres-exporter - -# Setup the Alembic configuration for the FastAPI application -cd .. -cd docker/minirag -cp alembic.example.ini alembic.ini - -### 2. Start the services - -```bash -cd docker -docker compose up --build -d -``` - -To start only specific services: - -```bash -docker compose up -d fastapi nginx pgvector qdrant -``` - -If you encounter connection issues, you may want to start the database services first and let them initialize before starting the application: - -```bash -# Start databases first -docker compose up -d pgvector qdrant postgres-exporter -# Wait for databases to be healthy -sleep 30 -# Start the application services -docker compose up fastapi nginx prometheus grafana node-exporter --build -d -``` - -In case deleting all containers and volumes is necessary, you can run: - -```bash -docker compose down -v --remove-orphans -``` - -### 3. Access the services - -- FastAPI Application: http://localhost:8000 -- FastAPI Documentation: http://localhost:8000/docs -- Nginx (serving FastAPI): http://localhost -- Prometheus: http://localhost:9090 -- Grafana: http://localhost:3000 -- Qdrant UI: http://localhost:6333/dashboard - -## Volume Management - -### Managing Docker Volumes - -Docker volumes are used to persist data generated by and used by Docker containers. Here are some commands to manage your volumes: - -1. **List all volumes**: - ```bash - docker volume ls - ``` -2. **Inspect a volume**: - ```bash - docker volume inspect - ``` - - - list files in a volume: - ```bash - docker run --rm -v :/data busybox ls -l /data - ``` - -3. **Remove a volume**: - ```bash - docker volume rm - ``` -4. **Prune unused volumes**: - ```bash - docker volume prune - ``` - -5. **Backup volume for migration**: - ```bash - docker run --rm -v :/volume -v $(pwd):/backup alpine tar cvf /backup/backup.tar /volume - ``` - -6. **Restore volume from backup**: - ```bash - docker run --rm -v :/volume -v $(pwd):/backup alpine sh -c "cd /volume && tar xvf /backup/backup.tar --strip 1" - ``` - -7. **Remove all volumes**: - ```bash - docker volume rm $(docker volume ls -q) - ``` - -**NOTE**: For PostgreSQL specifically, you might want to consider using PostgreSQL's built-in tools like `pg_dump` and `pg_restore` for more reliable backups, especially for live databases. - -## Monitoring - -### FastAPI Metrics - -FastAPI is configured to expose Prometheus metrics at the `/metrics` endpoint. These metrics include: - -- Request counts -- Request latencies -- Status codes - -Prometheus is configured to scrape these metrics automatically. - -### Visualizing Metrics in Grafana - -1. Log into Grafana at http://localhost:3000 (default credentials: admin/admin_password) -2. Add Prometheus as a data source (URL: http://prometheus:9090) -3. Import dashboards for FastAPI, PostgreSQL, and Qdrant - -#### Dashboards URLs - -https://grafana.com/grafana/dashboards/18739-fastapi-observability/ - -https://grafana.com/grafana/dashboards/1860-node-exporter-full/ - -https://grafana.com/grafana/dashboards/23033-qdrant/ - -https://grafana.com/grafana/dashboards/12485-postgresql-exporter/ - - -## Development Workflow - -The FastAPI application is configured with hot-reloading. Any changes to the code in the `src/` directory will automatically reload the application. - -## Troubleshooting - -### Connection Errors - -If you see connection errors when starting the services: - -1. **Database Connection Refused**: This often happens when the FastAPI app tries to connect to databases before they're ready. - ``` - Connection refused: [Errno 111] Connection refused - ``` - - Solutions: - - Start database services first, wait, then start the application - - Check database logs: `docker compose logs pgvector` - - Ensure your database credentials in `.env.app` match those in `.env.postgres` - -2. **Restart the FastAPI service** after databases are running: - ```bash - docker compose restart fastapi - ``` - -3. **Check service status**: - ```bash - docker compose ps - ``` - -4. **View logs** for more details: - ```bash - docker compose logs --tail=100 fastapi - docker compose logs --tail=100 pgvector - ``` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 8483a53b6..000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,250 +0,0 @@ -services: - # FastAPI Application - fastapi: - build: - context: .. - dockerfile: docker/minirag/Dockerfile - container_name: fastapi - ports: - - "8000:8000" - volumes: - - fastapi_data:/app/assets - networks: - - backend - restart: always - depends_on: - pgvector: - condition: service_healthy - env_file: - - ./env/.env.app - - # Celery Worker - celery-worker: - build: - context: .. - dockerfile: docker/minirag/Dockerfile - container_name: celery-worker - volumes: - - fastapi_data:/app/assets - networks: - - backend - restart: always - depends_on: - rabbitmq: - condition: service_healthy - redis: - condition: service_healthy - pgvector: - condition: service_healthy - env_file: - - ./env/.env.app - command: ["python", "-m", "celery", "-A", "celery_app", "worker", "--queues=default,file_processing,data_indexing", "--loglevel=info"] - - # Celery Beat Scheduler - celery-beat: - build: - context: .. - dockerfile: docker/minirag/Dockerfile - container_name: celery-beat - volumes: - - fastapi_data:/app/assets - - celery_beat_data:/app/celerybeat - networks: - - backend - restart: always - depends_on: - rabbitmq: - condition: service_healthy - redis: - condition: service_healthy - env_file: - - ./env/.env.app - command: ["python", "-m", "celery", "-A", "celery_app", "beat", "--loglevel=info"] - - # Flower Dashboard - flower: - build: - context: .. - dockerfile: docker/minirag/Dockerfile - container_name: flower - ports: - - "5555:5555" - networks: - - backend - restart: always - depends_on: - - rabbitmq - - celery-worker - env_file: - - ./env/.env.app - command: ["python", "-m", "celery", "-A", "celery_app", "flower", "--conf=flowerconfig.py"] - - # Nginx Service - nginx: - image: nginx:stable-alpine3.20-perl - container_name: nginx - ports: - - "80:80" - volumes: - - ./nginx/default.conf:/etc/nginx/conf.d/default.conf - depends_on: - - fastapi - networks: - - backend - restart: always - - - # PostgreSQL (pgvector) - pgvector: - image: pgvector/pgvector:0.8.0-pg17 - container_name: pgvector - ports: - - "5400:5432" - volumes: - - 'pgvector:/var/lib/postgresql/data' - env_file: - - ./env/.env.postgres - networks: - - backend - restart: always - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 10s - - # Qdrant (VectorDB) - qdrant: - image: qdrant/qdrant:v1.13.6 - container_name: qdrant - ports: - - "6333:6333" - - "6334:6334" - volumes: - - qdrant_data:/qdrant/storage - networks: - - backend - restart: always - - # Prometheus Monitoring - prometheus: - image: prom/prometheus:v3.3.0 - container_name: prometheus - ports: - - "9090:9090" - volumes: - - prometheus_data:/prometheus - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - networks: - - backend - restart: always - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--web.enable-lifecycle' - - # Grafana Dashboard - grafana: - image: grafana/grafana:11.6.0-ubuntu - container_name: grafana - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - env_file: - - ./env/.env.grafana - depends_on: - - prometheus - networks: - - backend - restart: always - - - # Node Exporter for system metrics - node-exporter: - image: prom/node-exporter:v1.9.1 - container_name: node-exporter - ports: - - "9100:9100" - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - '--path.procfs=/host/proc' - - '--path.rootfs=/rootfs' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - networks: - - backend - restart: always - - # PostgreSQL Exporter for Postgres metrics - postgres-exporter: - image: prometheuscommunity/postgres-exporter:v0.17.1 - container_name: postgres-exporter - ports: - - "9187:9187" - env_file: - - ./env/.env.postgres-exporter - depends_on: - - pgvector - networks: - - backend - restart: always - - # RabbitMQ (Message Broker) - rabbitmq: - image: rabbitmq:4.1.2-management-alpine - container_name: rabbitmq - ports: - - "5672:5672" # AMQP port - - "15672:15672" # Management UI port - volumes: - - rabbitmq_data:/var/lib/rabbitmq - - ./rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf - env_file: - - ./env/.env.rabbitmq - networks: - - backend - restart: always - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "ping"] - timeout: 10s - retries: 5 - - # Redis (Results Backend & Cache) - redis: - image: redis:8.0.3-alpine - container_name: redis - ports: - - "6379:6379" - volumes: - - redis_data:/data - env_file: - - ./env/.env.redis - networks: - - backend - restart: always - healthcheck: - test: ["CMD", "redis-cli", "ping"] - timeout: 10s - retries: 5 - command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD:-minirag_redis_2222}"] - -networks: - backend: - driver: bridge - -volumes: - fastapi_data: - pgvector: - qdrant_data: - prometheus_data: - grafana_data: - rabbitmq_data: - redis_data: - celery_beat_data: diff --git a/docker/env/.env.example.app b/docker/env/.env.example.app deleted file mode 100644 index fa576a30d..000000000 --- a/docker/env/.env.example.app +++ /dev/null @@ -1,49 +0,0 @@ -APP_NAME="mini-RAG" -APP_VERSION="0.1" - -FILE_ALLOWED_TYPES=["text/plain", "application/pdf"] -FILE_MAX_SIZE=10 -FILE_DEFAULT_CHUNK_SIZE=512000 # 512KB - -POSTGRES_USERNAME="postgres" -POSTGRES_PASSWORD="postgres_password" -POSTGRES_HOST="pgvector" -POSTGRES_PORT=5432 -POSTGRES_MAIN_DATABASE="minirag" - -# ========================= LLM Config ========================= -GENERATION_BACKEND = "OPENAI" -EMBEDDING_BACKEND = "COHERE" - -OPENAI_API_KEY="key___" -OPENAI_API_URL= "" -COHERE_API_KEY="key___" - -GENERATION_MODEL_ID_LITERAL = ["gpt-4o-mini", "gemma2:9b-instruct-q5_0"] -GENERATION_MODEL_ID="gpt-4o-mini" -EMBEDDING_MODEL_ID="embed-multilingual-v3.0" -EMBEDDING_MODEL_SIZE=1024 - -INPUT_DAFAULT_MAX_CHARACTERS=1024 -GENERATION_DAFAULT_MAX_TOKENS=200 -GENERATION_DAFAULT_TEMPERATURE=0.1 - -# ========================= Vector DB Config ========================= -VECTOR_DB_BACKEND_LITERAL = ["QDRANT", "PGVECTOR"] -VECTOR_DB_BACKEND = "PGVECTOR" -VECTOR_DB_PATH = "qdrant_db" -VECTOR_DB_DISTANCE_METHOD = "cosine" -VECTOR_DB_PGVEC_INDEX_THRESHOLD = 100 - -# ========================= Template Config ========================= -PRIMARY_LANG = "en" -DEFAULT_LANG = "en" - -# ========================= Celery Task Queue Config ========================= -CELERY_BROKER_URL="amqp://minirag_user:minirag_rabbitmq_2222@localhost:5672/minirag_vhost" -CELERY_RESULT_BACKEND="redis://:minirag_redis_2222@localhost:6379/0" -CELERY_TASK_SERIALIZER="json" -CELERY_TASK_TIME_LIMIT=600 -CELERY_TASK_ACKS_LATE=false -CELERY_WORKER_CONCURRENCY=2 -CELERY_FLOWER_PASSWORD="minirag_flower_2222" \ No newline at end of file diff --git a/docker/env/.env.example.grafana b/docker/env/.env.example.grafana deleted file mode 100644 index 127bedc9d..000000000 --- a/docker/env/.env.example.grafana +++ /dev/null @@ -1,4 +0,0 @@ -# Grafana Environment Variables -GF_SECURITY_ADMIN_USER=admin -GF_SECURITY_ADMIN_PASSWORD=admin_password -GF_USERS_ALLOW_SIGN_UP=false diff --git a/docker/env/.env.example.postgres b/docker/env/.env.example.postgres deleted file mode 100644 index cf4f155ef..000000000 --- a/docker/env/.env.example.postgres +++ /dev/null @@ -1,4 +0,0 @@ -# PostgreSQL Environment Variables -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres_password -POSTGRES_DB=minirag diff --git a/docker/env/.env.example.postgres-exporter b/docker/env/.env.example.postgres-exporter deleted file mode 100644 index 27483a275..000000000 --- a/docker/env/.env.example.postgres-exporter +++ /dev/null @@ -1,3 +0,0 @@ -DATA_SOURCE_URI=pgvector:5432/postgres?sslmode=disable -DATA_SOURCE_USER=postgres -DATA_SOURCE_PASS=postgres_password \ No newline at end of file diff --git a/docker/env/.env.example.rabbitmq b/docker/env/.env.example.rabbitmq deleted file mode 100644 index 9c8e11746..000000000 --- a/docker/env/.env.example.rabbitmq +++ /dev/null @@ -1,13 +0,0 @@ -# RabbitMQ Configuration Example -RABBITMQ_DEFAULT_USER=minirag_user -RABBITMQ_DEFAULT_PASS=minirag_rabbitmq_2222 -RABBITMQ_DEFAULT_VHOST=minirag_vhost - -# Management Plugin -RABBITMQ_MANAGEMENT_ENABLED=true - -# Security -RABBITMQ_AUTH_BACKENDS=rabbit_auth_backend_internal - -# Performance -RABBITMQ_DISK_FREE_LIMIT=2000000000 diff --git a/docker/env/.env.example.redis b/docker/env/.env.example.redis deleted file mode 100644 index 01ce9b386..000000000 --- a/docker/env/.env.example.redis +++ /dev/null @@ -1,12 +0,0 @@ -# Redis Configuration Example -REDIS_PASSWORD=minirag_redis_2222 - -# Persistence -REDIS_APPENDONLY=yes - -# Memory Management -REDIS_MAXMEMORY=512mb -REDIS_MAXMEMORY_POLICY=allkeys-lru - -# Security -REDIS_PROTECTED_MODE=yes diff --git a/docker/minirag.service b/docker/minirag.service deleted file mode 100644 index 6d998bd3f..000000000 --- a/docker/minirag.service +++ /dev/null @@ -1,23 +0,0 @@ -[Unit] -Description=MiniRAG Docker Service -After=network.target docker.service -Requires=docker.service - -[Service] -Type=forking -RemainAfterExit=yes -User=github_user -Group=docker -WorkingDirectory=/home/github_user/workspace/mini-rag/docker -ExecStartPre=/bin/bash -c '/usr/bin/docker compose down || true' -ExecStartPre=/bin/sleep 5 -ExecStart=/usr/bin/docker compose up --build -d -ExecStop=/usr/bin/docker compose down -ExecReload=/usr/bin/docker compose restart -TimeoutStartSec=300 -TimeoutStopSec=120 -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/docker/minirag/Dockerfile b/docker/minirag/Dockerfile deleted file mode 100644 index 7447fbab8..000000000 --- a/docker/minirag/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM ghcr.io/astral-sh/uv:0.6.14-python3.10-bookworm - -WORKDIR /app - -# Install additional system dependencies for lxml and other packages -RUN apt-get update && apt-get install -y \ - build-essential \ - libavif-dev pkg-config \ - libjpeg-dev \ - gcc unzip zip \ - python3-dev \ - libxml2-dev \ - libxslt1-dev \ - libffi-dev \ - curl \ - && rm -rf /var/lib/apt/lists/* - -COPY src/requirements.txt . - -RUN uv pip install -r requirements.txt --system - -COPY src/ . - -# Create directory structure for Alembic -RUN mkdir -p /app/models/db_schemes/minirag/ - -COPY docker/minirag/alembic.ini /app/models/db_schemes/minirag/alembic.ini - -COPY docker/minirag/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] - -# Command to run the application -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"] diff --git a/docker/minirag/alembic.example.ini b/docker/minirag/alembic.example.ini deleted file mode 100644 index 346ec24f9..000000000 --- a/docker/minirag/alembic.example.ini +++ /dev/null @@ -1,117 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -# Use forward slashes (/) also on windows to provide an os agnostic path -script_location = alembic - -# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s -# Uncomment the line below if you want the files to be prepended with date and time -# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file -# for all available tokens -# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s - -# sys.path path, will be prepended to sys.path if present. -# defaults to the current working directory. -prepend_sys_path = . - -# timezone to use when rendering the date within the migration file -# as well as the filename. -# If specified, requires the python>=3.9 or backports.zoneinfo library. -# Any required deps can installed by adding `alembic[tz]` to the pip requirements -# string value is passed to ZoneInfo() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; This defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path. -# The path separator used here should be the separator specified by "version_path_separator" below. -# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions - -# version path separator; As mentioned above, this is the character used to split -# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. -# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. -# Valid values for version_path_separator are: -# -# version_path_separator = : -# version_path_separator = ; -# version_path_separator = space -# version_path_separator = newline -version_path_separator = os # Use os.pathsep. Default configuration used for new projects. - -# set to 'true' to search source files recursively -# in each "version_locations" directory -# new in Alembic version 1.10 -# recursive_version_locations = false - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -sqlalchemy.url = postgresql://postgres:postgres_password@pgvector:5432/minirag - - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME - -# lint with attempts to fix using "ruff" - use the exec runner, execute a binary -# hooks = ruff -# ruff.type = exec -# ruff.executable = %(here)s/.venv/bin/ruff -# ruff.options = --fix REVISION_SCRIPT_FILENAME - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARNING -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARNING -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/docker/minirag/entrypoint.sh b/docker/minirag/entrypoint.sh deleted file mode 100644 index 198c8cb04..000000000 --- a/docker/minirag/entrypoint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -echo "Running database migrations..." -cd /app/models/db_schemes/minirag/ -alembic upgrade head -cd /app diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf deleted file mode 100644 index 87632089f..000000000 --- a/docker/nginx/default.conf +++ /dev/null @@ -1,17 +0,0 @@ -server { - listen 80; - server_name localhost; - - location / { - proxy_pass http://fastapi:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # Optionally expose metrics endpoint directly - location /TrhBVe_m5gg2002_E5VVqS { - proxy_pass http://fastapi:8000/TrhBVe_m5gg2002_E5VVqS; - } -} \ No newline at end of file diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml deleted file mode 100644 index 802caa6dd..000000000 --- a/docker/prometheus/prometheus.yml +++ /dev/null @@ -1,27 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - - job_name: 'fastapi' - static_configs: - - targets: ['fastapi:8000'] - metrics_path: '/TrhBVe_m5gg2002_E5VVqS' - - - job_name: 'node-exporter' - static_configs: - - targets: ['node-exporter:9100'] - - - job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] - - - job_name: 'qdrant' - static_configs: - - targets: ['qdrant:6333'] - metrics_path: '/metrics' - - - job_name: 'postgres' - static_configs: - - targets: ['postgres-exporter:9187'] diff --git a/docker/rabbitmq/rabbitmq.conf b/docker/rabbitmq/rabbitmq.conf deleted file mode 100644 index 5cf61d577..000000000 --- a/docker/rabbitmq/rabbitmq.conf +++ /dev/null @@ -1,18 +0,0 @@ -# RabbitMQ Configuration File - -# Memory management -vm_memory_high_watermark.relative = 0.6 - -# Disk space management -disk_free_limit.absolute = 2GB - -# SSL/TLS configuration -ssl_options.verify = verify_none - -# Management plugin -management.tcp.port = 15672 - -# Logging -log.file.level = info -log.console = true -log.console.level = info diff --git a/src/.env.example b/src/.env.example deleted file mode 100644 index 402095884..000000000 --- a/src/.env.example +++ /dev/null @@ -1,50 +0,0 @@ -APP_NAME="mini-RAG" -APP_VERSION="0.1" -OPENAI_API_KEY="sk-" - -FILE_ALLOWED_TYPES=["text/plain", "application/pdf"] -FILE_MAX_SIZE=10 -FILE_DEFAULT_CHUNK_SIZE=512000 # 512KB - -POSTGRES_USERNAME="postgres" -POSTGRES_PASSWORD="minirag2222" -POSTGRES_HOST="localhost" -POSTGRES_PORT=5432 -POSTGRES_MAIN_DATABASE="minirag" - -# ========================= LLM Config ========================= -GENERATION_BACKEND = "OPENAI" -EMBEDDING_BACKEND = "COHERE" - -OPENAI_API_KEY="sk-" -OPENAI_API_URL= -COHERE_API_KEY="m8-" - -GENERATION_MODEL_ID_LITERAL = ["gpt-4o-mini", "gpt-4o"] -GENERATION_MODEL_ID="gpt-4o-mini" -EMBEDDING_MODEL_ID="embed-multilingual-light-v3.0" -EMBEDDING_MODEL_SIZE=384 - -INPUT_DAFAULT_MAX_CHARACTERS=1024 -GENERATION_DAFAULT_MAX_TOKENS=200 -GENERATION_DAFAULT_TEMPERATURE=0.1 - -# ========================= Vector DB Config ========================= -VECTOR_DB_BACKEND_LITERAL = ["QDRANT", "PGVECTOR"] -VECTOR_DB_BACKEND = "PGVECTOR" -VECTOR_DB_PATH = "qdrant_db" -VECTOR_DB_DISTANCE_METHOD = "cosine" -VECTOR_DB_PGVEC_INDEX_THRESHOLD = - -# ========================= Template Configs ========================= -PRIMARY_LANG = "ar" -DEFAULT_LANG = "en" - -# ========================= Celery Task Queue Config ========================= -CELERY_BROKER_URL="amqp://minirag_user:minirag_rabbitmq_2222@localhost:5672/minirag_vhost" -CELERY_RESULT_BACKEND="redis://:minirag_redis_2222@localhost:6379/0" -CELERY_TASK_SERIALIZER="json" -CELERY_TASK_TIME_LIMIT=600 -CELERY_TASK_ACKS_LATE=false -CELERY_WORKER_CONCURRENCY=2 -CELERY_FLOWER_PASSWORD="minirag_flower_2222" diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index 3069115fd..000000000 --- a/src/.gitignore +++ /dev/null @@ -1,163 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so -*.bak -*.dat -*.dir - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ diff --git a/src/assets/.gitignore b/src/assets/.gitignore deleted file mode 100644 index ac32e2b61..000000000 --- a/src/assets/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -files -database diff --git a/src/assets/.gitkeep b/src/assets/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/assets/mini-rag-app.postman_collection.json b/src/assets/mini-rag-app.postman_collection.json deleted file mode 100644 index 2d58f447e..000000000 --- a/src/assets/mini-rag-app.postman_collection.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "info": { - "_postman_id": "1f67dedb-1b93-4639-9f96-7fe8681693f4", - "name": "mini-rag-app", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "854486", - "_collection_link": "https://www.postman.com/gold-water-645258/workspace/mini-rag-app/collection/854486-1f67dedb-1b93-4639-9f96-7fe8681693f4?action=share&source=collection_link&creator=854486" - }, - "item": [ - { - "name": "welcome-request", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{api}}/welcome", - "host": [ - "{{api}}" - ], - "path": [ - "welcome" - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "" - ] - } - } - ], - "variable": [ - { - "key": "api", - "value": "http://127.0.0.1:8000", - "type": "string" - } - ] -} \ No newline at end of file diff --git a/src/celery_app.py b/src/celery_app.py deleted file mode 100644 index 4831ab8e5..000000000 --- a/src/celery_app.py +++ /dev/null @@ -1,107 +0,0 @@ -from celery import Celery -from helpers.config import get_settings - -from stores.llm.LLMProviderFactory import LLMProviderFactory -from stores.vectordb.VectorDBProviderFactory import VectorDBProviderFactory -from stores.llm.templates.template_parser import TemplateParser -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from sqlalchemy.orm import sessionmaker - -settings = get_settings() - -async def get_setup_utils(): - settings = get_settings() - - postgres_conn = f"postgresql+asyncpg://{settings.POSTGRES_USERNAME}:{settings.POSTGRES_PASSWORD}@{settings.POSTGRES_HOST}:{settings.POSTGRES_PORT}/{settings.POSTGRES_MAIN_DATABASE}" - - db_engine = create_async_engine(postgres_conn) - db_client = sessionmaker( - db_engine, class_=AsyncSession, expire_on_commit=False - ) - - llm_provider_factory = LLMProviderFactory(settings) - vectordb_provider_factory = VectorDBProviderFactory(config=settings, db_client=db_client) - - # generation client - generation_client = llm_provider_factory.create(provider=settings.GENERATION_BACKEND) - generation_client.set_generation_model(model_id = settings.GENERATION_MODEL_ID) - - # embedding client - embedding_client = llm_provider_factory.create(provider=settings.EMBEDDING_BACKEND) - embedding_client.set_embedding_model(model_id=settings.EMBEDDING_MODEL_ID, - embedding_size=settings.EMBEDDING_MODEL_SIZE) - - # vector db client - vectordb_client = vectordb_provider_factory.create( - provider=settings.VECTOR_DB_BACKEND - ) - await vectordb_client.connect() - - template_parser = TemplateParser( - language=settings.PRIMARY_LANG, - default_language=settings.DEFAULT_LANG, - ) - - return (db_engine, db_client, llm_provider_factory, vectordb_provider_factory, - generation_client, embedding_client, vectordb_client, template_parser) - -# Create Celery application instance -celery_app = Celery( - "minirag", - broker=settings.CELERY_BROKER_URL, - backend=settings.CELERY_RESULT_BACKEND, - include=[ - "tasks.file_processing", - "tasks.data_indexing", - "tasks.process_workflow", - "tasks.maintenance", - ] -) - -# Configure Celery with essential settings -celery_app.conf.update( - task_serializer=settings.CELERY_TASK_SERIALIZER, - result_serializer=settings.CELERY_TASK_SERIALIZER, - accept_content=[ - settings.CELERY_TASK_SERIALIZER - ], - - # Task safety - Late acknowledgment prevents task loss on worker crash - task_acks_late=settings.CELERY_TASK_ACKS_LATE, - - # Time limits - Prevent hanging tasks - task_time_limit=settings.CELERY_TASK_TIME_LIMIT, - - # Result backend - Store results for status tracking - task_ignore_resul=False, - result_expires=3600, - - # Worker settings - worker_concurrency=settings.CELERY_WORKER_CONCURRENCY, - - # Connection settings for better reliability - broker_connection_retry_on_startup=True, - broker_connection_retry=True, - broker_connection_max_retries=10, - worker_cancel_long_running_tasks_on_connection_loss=True, - - task_routes={ - "tasks.file_processing.process_project_files": {"queue": "file_processing"}, - "tasks.data_indexing.index_data_content": {"queue": "data_indexing"}, - "tasks.process_workflow.process_and_push_workflow": {"queue": "file_processing"}, - "tasks.maintenance.clean_celery_executions_table": {"queue": "default"}, - }, - - beat_schedule={ - 'cleanup-old-task-records': { - 'task': "tasks.maintenance.clean_celery_executions_table", - 'schedule': 10, - 'args': () - } - }, - - timezone='UTC', - -) - -celery_app.conf.task_default_queue = "default" \ No newline at end of file diff --git a/src/controllers/BaseController.py b/src/controllers/BaseController.py deleted file mode 100644 index aa3e573d9..000000000 --- a/src/controllers/BaseController.py +++ /dev/null @@ -1,35 +0,0 @@ -from helpers.config import get_settings, Settings -import os -import random -import string - -class BaseController: - - def __init__(self): - - self.app_settings = get_settings() - - self.base_dir = os.path.dirname( os.path.dirname(__file__) ) - self.files_dir = os.path.join( - self.base_dir, - "assets/files" - ) - - self.database_dir = os.path.join( - self.base_dir, - "assets/database" - ) - - def generate_random_string(self, length: int=12): - return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length)) - - def get_database_path(self, db_name: str): - - database_path = os.path.join( - self.database_dir, db_name - ) - - if not os.path.exists(database_path): - os.makedirs(database_path) - - return database_path \ No newline at end of file diff --git a/src/controllers/DataController.py b/src/controllers/DataController.py deleted file mode 100644 index d4bd5aab3..000000000 --- a/src/controllers/DataController.py +++ /dev/null @@ -1,57 +0,0 @@ -from .BaseController import BaseController -from .ProjectController import ProjectController -from fastapi import UploadFile -from models import ResponseSignal -import re -import os - -class DataController(BaseController): - - def __init__(self): - super().__init__() - self.size_scale = 1048576 # convert MB to bytes - - def validate_uploaded_file(self, file: UploadFile): - - if file.content_type not in self.app_settings.FILE_ALLOWED_TYPES: - return False, ResponseSignal.FILE_TYPE_NOT_SUPPORTED.value - - if file.size > self.app_settings.FILE_MAX_SIZE * self.size_scale: - return False, ResponseSignal.FILE_SIZE_EXCEEDED.value - - return True, ResponseSignal.FILE_VALIDATED_SUCCESS.value - - def generate_unique_filepath(self, orig_file_name: str, project_id: str): - - random_key = self.generate_random_string() - project_path = ProjectController().get_project_path(project_id=project_id) - - cleaned_file_name = self.get_clean_file_name( - orig_file_name=orig_file_name - ) - - new_file_path = os.path.join( - project_path, - random_key + "_" + cleaned_file_name - ) - - while os.path.exists(new_file_path): - random_key = self.generate_random_string() - new_file_path = os.path.join( - project_path, - random_key + "_" + cleaned_file_name - ) - - return new_file_path, random_key + "_" + cleaned_file_name - - def get_clean_file_name(self, orig_file_name: str): - - # remove any special characters, except underscore and . - cleaned_file_name = re.sub(r'[^\w.]', '', orig_file_name.strip()) - - # replace spaces with underscore - cleaned_file_name = cleaned_file_name.replace(" ", "_") - - return cleaned_file_name - - diff --git a/src/controllers/NLPController.py b/src/controllers/NLPController.py deleted file mode 100644 index c2e490955..000000000 --- a/src/controllers/NLPController.py +++ /dev/null @@ -1,141 +0,0 @@ -from .BaseController import BaseController -from models.db_schemes import Project, DataChunk -from stores.llm.LLMEnums import DocumentTypeEnum -from typing import List -import json - -class NLPController(BaseController): - - def __init__(self, vectordb_client, generation_client, - embedding_client, template_parser): - super().__init__() - - self.vectordb_client = vectordb_client - self.generation_client = generation_client - self.embedding_client = embedding_client - self.template_parser = template_parser - - def create_collection_name(self, project_id: str): - return f"collection_{self.vectordb_client.default_vector_size}_{project_id}".strip() - - async def reset_vector_db_collection(self, project: Project): - collection_name = self.create_collection_name(project_id=project.project_id) - return await self.vectordb_client.delete_collection(collection_name=collection_name) - - async def get_vector_db_collection_info(self, project: Project): - collection_name = self.create_collection_name(project_id=project.project_id) - collection_info = await self.vectordb_client.get_collection_info(collection_name=collection_name) - - return json.loads( - json.dumps(collection_info, default=lambda x: x.__dict__) - ) - - async def index_into_vector_db(self, project: Project, chunks: List[DataChunk], - chunks_ids: List[int], - do_reset: bool = False): - - # step1: get collection name - collection_name = self.create_collection_name(project_id=project.project_id) - - # step2: manage items - texts = [ c.chunk_text for c in chunks ] - metadata = [ c.chunk_metadata for c in chunks] - vectors = self.embedding_client.embed_text(text=texts, - document_type=DocumentTypeEnum.DOCUMENT.value) - - # step3: create collection if not exists - _ = await self.vectordb_client.create_collection( - collection_name=collection_name, - embedding_size=self.embedding_client.embedding_size, - do_reset=do_reset, - ) - - # step4: insert into vector db - _ = await self.vectordb_client.insert_many( - collection_name=collection_name, - texts=texts, - metadata=metadata, - vectors=vectors, - record_ids=chunks_ids, - ) - - return True - - async def search_vector_db_collection(self, project: Project, text: str, limit: int = 10): - - # step1: get collection name - query_vector = None - collection_name = self.create_collection_name(project_id=project.project_id) - - # step2: get text embedding vector - vectors = self.embedding_client.embed_text(text=text, - document_type=DocumentTypeEnum.QUERY.value) - - if not vectors or len(vectors) == 0: - return False - - if isinstance(vectors, list) and len(vectors) > 0: - query_vector = vectors[0] - - if not query_vector: - return False - - # step3: do semantic search - results = await self.vectordb_client.search_by_vector( - collection_name=collection_name, - vector=query_vector, - limit=limit - ) - - if not results: - return False - - return results - - async def answer_rag_question(self, project: Project, query: str, limit: int = 10): - - answer, full_prompt, chat_history = None, None, None - - # step1: retrieve related documents - retrieved_documents = await self.search_vector_db_collection( - project=project, - text=query, - limit=limit, - ) - - if not retrieved_documents or len(retrieved_documents) == 0: - return answer, full_prompt, chat_history - - # step2: Construct LLM prompt - system_prompt = self.template_parser.get("rag", "system_prompt") - - documents_prompts = "\n".join([ - self.template_parser.get("rag", "document_prompt", { - "doc_num": idx + 1, - "chunk_text": self.generation_client.process_text(doc.text), - }) - for idx, doc in enumerate(retrieved_documents) - ]) - - footer_prompt = self.template_parser.get("rag", "footer_prompt", { - "query": query - }) - - # step3: Construct Generation Client Prompts - chat_history = [ - self.generation_client.construct_prompt( - prompt=system_prompt, - role=self.generation_client.enums.SYSTEM.value, - ) - ] - - full_prompt = "\n\n".join([ documents_prompts, footer_prompt]) - - # step4: Retrieve the Answer - answer = self.generation_client.generate_text( - prompt=full_prompt, - chat_history=chat_history - ) - - return answer, full_prompt, chat_history - diff --git a/src/controllers/ProcessController.py b/src/controllers/ProcessController.py deleted file mode 100644 index ecaa08774..000000000 --- a/src/controllers/ProcessController.py +++ /dev/null @@ -1,109 +0,0 @@ -from .BaseController import BaseController -from .ProjectController import ProjectController -import os -from langchain_community.document_loaders import TextLoader -from langchain_community.document_loaders import PyMuPDFLoader -from models import ProcessingEnum -from typing import List -from dataclasses import dataclass - -@dataclass -class Document: - page_content: str - metadata: dict - -class ProcessController(BaseController): - - def __init__(self, project_id: str): - super().__init__() - - self.project_id = project_id - self.project_path = ProjectController().get_project_path(project_id=project_id) - - def get_file_extension(self, file_id: str): - return os.path.splitext(file_id)[-1] - - def get_file_loader(self, file_id: str): - - file_ext = self.get_file_extension(file_id=file_id) - file_path = os.path.join( - self.project_path, - file_id - ) - - if not os.path.exists(file_path): - return None - - if file_ext == ProcessingEnum.TXT.value: - return TextLoader(file_path, encoding="utf-8") - - if file_ext == ProcessingEnum.PDF.value: - return PyMuPDFLoader(file_path) - - return None - - def get_file_content(self, file_id: str): - - loader = self.get_file_loader(file_id=file_id) - if loader: - return loader.load() - - return None - - def process_file_content(self, file_content: list, file_id: str, - chunk_size: int=100, overlap_size: int=20): - - file_content_texts = [ - rec.page_content - for rec in file_content - ] - - file_content_metadata = [ - rec.metadata - for rec in file_content - ] - - # chunks = text_splitter.create_documents( - # file_content_texts, - # metadatas=file_content_metadata - # ) - - chunks = self.process_simpler_splitter( - texts=file_content_texts, - metadatas=file_content_metadata, - chunk_size=chunk_size, - ) - - return chunks - - def process_simpler_splitter(self, texts: List[str], metadatas: List[dict], chunk_size: int, splitter_tag: str="\n"): - - full_text = " ".join(texts) - - # split by splitter_tag - lines = [ doc.strip() for doc in full_text.split(splitter_tag) if len(doc.strip()) > 1 ] - - chunks = [] - current_chunk = "" - - for line in lines: - current_chunk += line + splitter_tag - if len(current_chunk) >= chunk_size: - chunks.append(Document( - page_content=current_chunk.strip(), - metadata={} - )) - - current_chunk = "" - - if len(current_chunk) >= 0: - chunks.append(Document( - page_content=current_chunk.strip(), - metadata={} - )) - - return chunks - - - - diff --git a/src/controllers/ProjectController.py b/src/controllers/ProjectController.py deleted file mode 100644 index b3105a57c..000000000 --- a/src/controllers/ProjectController.py +++ /dev/null @@ -1,22 +0,0 @@ -from .BaseController import BaseController -from fastapi import UploadFile -from models import ResponseSignal -import os - -class ProjectController(BaseController): - - def __init__(self): - super().__init__() - - def get_project_path(self, project_id: str): - project_dir = os.path.join( - self.files_dir, - str(project_id) - ) - - if not os.path.exists(project_dir): - os.makedirs(project_dir) - - return project_dir - - diff --git a/src/controllers/__init__.py b/src/controllers/__init__.py deleted file mode 100644 index 8876467ad..000000000 --- a/src/controllers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .DataController import DataController -from .ProjectController import ProjectController -from .ProcessController import ProcessController -from .NLPController import NLPController - diff --git a/src/flowerconfig.py b/src/flowerconfig.py deleted file mode 100644 index 5bebec51c..000000000 --- a/src/flowerconfig.py +++ /dev/null @@ -1,12 +0,0 @@ -from dotenv import dotenv_values -config = dotenv_values(".env") - -# Flower configuration -port = 5555 -max_tasks = 10000 -# db = 'flower.db' # SQLite database for persistent storage -auto_refresh = True - -# Authentication (optional) -basic_auth = [f'admin:{config["CELERY_FLOWER_PASSWORD"]}'] - diff --git a/src/helpers/__init__.py b/src/helpers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/helpers/config.py b/src/helpers/config.py deleted file mode 100644 index fb8600b8a..000000000 --- a/src/helpers/config.py +++ /dev/null @@ -1,57 +0,0 @@ -from pydantic_settings import BaseSettings, SettingsConfigDict -from typing import List - -class Settings(BaseSettings): - - APP_NAME: str - APP_VERSION: str - OPENAI_API_KEY: str - - FILE_ALLOWED_TYPES: list - FILE_MAX_SIZE: int - FILE_DEFAULT_CHUNK_SIZE: int - - POSTGRES_USERNAME: str - POSTGRES_PASSWORD: str - POSTGRES_HOST: str - POSTGRES_PORT: int - POSTGRES_MAIN_DATABASE: str - - GENERATION_BACKEND: str - EMBEDDING_BACKEND: str - - OPENAI_API_KEY: str = None - OPENAI_API_URL: str = None - COHERE_API_KEY: str = None - - GENERATION_MODEL_ID_LITERAL: List[str] = None - GENERATION_MODEL_ID: str = None - EMBEDDING_MODEL_ID: str = None - EMBEDDING_MODEL_SIZE: int = None - INPUT_DAFAULT_MAX_CHARACTERS: int = None - GENERATION_DAFAULT_MAX_TOKENS: int = None - GENERATION_DAFAULT_TEMPERATURE: float = None - - VECTOR_DB_BACKEND_LITERAL: List[str] = None - VECTOR_DB_BACKEND : str - VECTOR_DB_PATH : str - VECTOR_DB_DISTANCE_METHOD: str = None - VECTOR_DB_PGVEC_INDEX_THRESHOLD: int = 100 - - PRIMARY_LANG: str = "en" - DEFAULT_LANG: str = "en" - - # Celery Configuration - CELERY_BROKER_URL: str = None - CELERY_RESULT_BACKEND: str = None - CELERY_TASK_SERIALIZER: str = "json" - CELERY_TASK_TIME_LIMIT: int = 600 - CELERY_TASK_ACKS_LATE: bool = True - CELERY_WORKER_CONCURRENCY: int = 2 - CELERY_FLOWER_PASSWORD: str = None - - class Config: - env_file = ".env" - -def get_settings(): - return Settings() diff --git a/src/main.py b/src/main.py deleted file mode 100644 index ffd2c2e38..000000000 --- a/src/main.py +++ /dev/null @@ -1,61 +0,0 @@ -from fastapi import FastAPI -from routes import base, data, nlp -from helpers.config import get_settings -from stores.llm.LLMProviderFactory import LLMProviderFactory -from stores.vectordb.VectorDBProviderFactory import VectorDBProviderFactory -from stores.llm.templates.template_parser import TemplateParser -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from sqlalchemy.orm import sessionmaker - -# Import metrics setup -from utils.metrics import setup_metrics - -app = FastAPI() - -# Setup Prometheus metrics -setup_metrics(app) - -async def startup_span(): - settings = get_settings() - - postgres_conn = f"postgresql+asyncpg://{settings.POSTGRES_USERNAME}:{settings.POSTGRES_PASSWORD}@{settings.POSTGRES_HOST}:{settings.POSTGRES_PORT}/{settings.POSTGRES_MAIN_DATABASE}" - - app.db_engine = create_async_engine(postgres_conn) - app.db_client = sessionmaker( - app.db_engine, class_=AsyncSession, expire_on_commit=False - ) - - llm_provider_factory = LLMProviderFactory(settings) - vectordb_provider_factory = VectorDBProviderFactory(config=settings, db_client=app.db_client) - - # generation client - app.generation_client = llm_provider_factory.create(provider=settings.GENERATION_BACKEND) - app.generation_client.set_generation_model(model_id = settings.GENERATION_MODEL_ID) - - # embedding client - app.embedding_client = llm_provider_factory.create(provider=settings.EMBEDDING_BACKEND) - app.embedding_client.set_embedding_model(model_id=settings.EMBEDDING_MODEL_ID, - embedding_size=settings.EMBEDDING_MODEL_SIZE) - - # vector db client - app.vectordb_client = vectordb_provider_factory.create( - provider=settings.VECTOR_DB_BACKEND - ) - await app.vectordb_client.connect() - - app.template_parser = TemplateParser( - language=settings.PRIMARY_LANG, - default_language=settings.DEFAULT_LANG, - ) - - -async def shutdown_span(): - app.db_engine.dispose() - await app.vectordb_client.disconnect() - -app.on_event("startup")(startup_span) -app.on_event("shutdown")(shutdown_span) - -app.include_router(base.base_router) -app.include_router(data.data_router) -app.include_router(nlp.nlp_router) diff --git a/src/models/AssetModel.py b/src/models/AssetModel.py deleted file mode 100644 index 594e3a96f..000000000 --- a/src/models/AssetModel.py +++ /dev/null @@ -1,50 +0,0 @@ -from .BaseDataModel import BaseDataModel -from .db_schemes import Asset -from .enums.DataBaseEnum import DataBaseEnum -from bson import ObjectId -from sqlalchemy.future import select - -class AssetModel(BaseDataModel): - - def __init__(self, db_client: object): - super().__init__(db_client=db_client) - self.db_client = db_client - - @classmethod - async def create_instance(cls, db_client: object): - instance = cls(db_client) - return instance - - async def create_asset(self, asset: Asset): - - async with self.db_client() as session: - async with session.begin(): - session.add(asset) - await session.commit() - await session.refresh(asset) - return asset - - async def get_all_project_assets(self, asset_project_id: str, asset_type: str): - - async with self.db_client() as session: - stmt = select(Asset).where( - Asset.asset_project_id == asset_project_id, - Asset.asset_type == asset_type - ) - result = await session.execute(stmt) - records = result.scalars().all() - return records - - async def get_asset_record(self, asset_project_id: str, asset_name: str): - - async with self.db_client() as session: - stmt = select(Asset).where( - Asset.asset_project_id == asset_project_id, - Asset.asset_name == asset_name - ) - result = await session.execute(stmt) - record = result.scalar_one_or_none() - return record - - - diff --git a/src/models/BaseDataModel.py b/src/models/BaseDataModel.py deleted file mode 100644 index ab9bace10..000000000 --- a/src/models/BaseDataModel.py +++ /dev/null @@ -1,7 +0,0 @@ -from helpers.config import get_settings, Settings - -class BaseDataModel: - - def __init__(self, db_client: object): - self.db_client = db_client - self.app_settings = get_settings() diff --git a/src/models/ChunkModel.py b/src/models/ChunkModel.py deleted file mode 100644 index 8768d5fc5..000000000 --- a/src/models/ChunkModel.py +++ /dev/null @@ -1,69 +0,0 @@ -from .BaseDataModel import BaseDataModel -from .db_schemes import DataChunk -from .enums.DataBaseEnum import DataBaseEnum -from bson.objectid import ObjectId -from pymongo import InsertOne -from sqlalchemy.future import select -from sqlalchemy import func, delete - -class ChunkModel(BaseDataModel): - - def __init__(self, db_client: object): - super().__init__(db_client=db_client) - self.db_client = db_client - - @classmethod - async def create_instance(cls, db_client: object): - instance = cls(db_client) - return instance - - async def create_chunk(self, chunk: DataChunk): - - async with self.db_client() as session: - async with session.begin(): - session.add(chunk) - await session.commit() - await session.refresh(chunk) - return chunk - - async def get_chunk(self, chunk_id: str): - - async with self.db_client() as session: - result = await session.execute(select(DataChunk).where(DataChunk.chunk_id == chunk_id)) - chunk = result.scalar_one_or_none() - return chunk - - async def insert_many_chunks(self, chunks: list, batch_size: int=100): - - async with self.db_client() as session: - async with session.begin(): - for i in range(0, len(chunks), batch_size): - batch = chunks[i:i+batch_size] - session.add_all(batch) - await session.commit() - return len(chunks) - - async def delete_chunks_by_project_id(self, project_id: ObjectId): - async with self.db_client() as session: - stmt = delete(DataChunk).where(DataChunk.chunk_project_id == project_id) - result = await session.execute(stmt) - await session.commit() - return result.rowcount - - async def get_poject_chunks(self, project_id: ObjectId, page_no: int=1, page_size: int=50): - async with self.db_client() as session: - stmt = select(DataChunk).where(DataChunk.chunk_project_id == project_id).offset((page_no - 1) * page_size).limit(page_size) - result = await session.execute(stmt) - records = result.scalars().all() - return records - - async def get_total_chunks_count(self, project_id: ObjectId): - total_count = 0 - async with self.db_client() as session: - count_sql = select(func.count(DataChunk.chunk_id)).where(DataChunk.chunk_project_id == project_id) - records_count = await session.execute(count_sql) - total_count = records_count.scalar() - - return total_count - - diff --git a/src/models/ProjectModel.py b/src/models/ProjectModel.py deleted file mode 100644 index c3342af8d..000000000 --- a/src/models/ProjectModel.py +++ /dev/null @@ -1,61 +0,0 @@ -from .BaseDataModel import BaseDataModel -from .db_schemes import Project -from .enums.DataBaseEnum import DataBaseEnum -from sqlalchemy.future import select -from sqlalchemy import func - -class ProjectModel(BaseDataModel): - - def __init__(self, db_client: object): - super().__init__(db_client=db_client) - self.db_client = db_client - - @classmethod - async def create_instance(cls, db_client: object): - instance = cls(db_client) - return instance - - async def create_project(self, project: Project): - async with self.db_client() as session: - async with session.begin(): - session.add(project) - await session.commit() - await session.refresh(project) - - return project - - async def get_project_or_create_one(self, project_id: str): - async with self.db_client() as session: - async with session.begin(): - query = select(Project).where(Project.project_id == project_id) - result = await session.execute(query) - project = result.scalar_one_or_none() - if project is None: - project_rec = Project( - project_id = project_id - ) - - project = await self.create_project(project=project_rec) - return project - else: - return project - - async def get_all_projects(self, page: int=1, page_size: int=10): - - async with self.db_client() as session: - async with session.begin(): - - total_documents = await session.execute(select( - func.count( Project.project_id ) - )) - - total_documents = total_documents.scalar_one() - - total_pages = total_documents // page_size - if total_documents % page_size > 0: - total_pages += 1 - - query = select(Project).offset((page - 1) * page_size ).limit(page_size) - projects = await session.execute(query).scalars().all() - - return projects, total_pages diff --git a/src/models/__init__.py b/src/models/__init__.py deleted file mode 100644 index 3b41f8a2c..000000000 --- a/src/models/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .enums.ResponseEnums import ResponseSignal -from .enums.ProcessingEnum import ProcessingEnum - diff --git a/src/models/db_schemes/__init__.py b/src/models/db_schemes/__init__.py deleted file mode 100644 index e5301c78a..000000000 --- a/src/models/db_schemes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from models.db_schemes.minirag.schemes import Project, DataChunk, Asset, RetrievedDocument diff --git a/src/models/db_schemes/minirag/.gitignore b/src/models/db_schemes/minirag/.gitignore deleted file mode 100644 index a2462558a..000000000 --- a/src/models/db_schemes/minirag/.gitignore +++ /dev/null @@ -1 +0,0 @@ -alembic.ini diff --git a/src/models/db_schemes/minirag/README.md b/src/models/db_schemes/minirag/README.md deleted file mode 100644 index a9437d7fa..000000000 --- a/src/models/db_schemes/minirag/README.md +++ /dev/null @@ -1,21 +0,0 @@ -## Run Alembic Migrations - -### Configuration - -```bash -cp alembic.ini.example alembic.ini -``` - -- Update the `alembic.ini` with your database credentials (`sqlalchemy.url`) - -### (Optional) Create a new migration - -```bash -alembic revision --autogenerate -m "Add ..." -``` - -### Upgrade the database - -```bash -alembic upgrade head -``` diff --git a/src/models/db_schemes/minirag/__init__.py b/src/models/db_schemes/minirag/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/models/db_schemes/minirag/alembic.ini.example b/src/models/db_schemes/minirag/alembic.ini.example deleted file mode 100644 index 0e50bdb1e..000000000 --- a/src/models/db_schemes/minirag/alembic.ini.example +++ /dev/null @@ -1,117 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -# Use forward slashes (/) also on windows to provide an os agnostic path -script_location = alembic - -# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s -# Uncomment the line below if you want the files to be prepended with date and time -# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file -# for all available tokens -# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s - -# sys.path path, will be prepended to sys.path if present. -# defaults to the current working directory. -prepend_sys_path = . - -# timezone to use when rendering the date within the migration file -# as well as the filename. -# If specified, requires the python>=3.9 or backports.zoneinfo library. -# Any required deps can installed by adding `alembic[tz]` to the pip requirements -# string value is passed to ZoneInfo() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; This defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path. -# The path separator used here should be the separator specified by "version_path_separator" below. -# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions - -# version path separator; As mentioned above, this is the character used to split -# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. -# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. -# Valid values for version_path_separator are: -# -# version_path_separator = : -# version_path_separator = ; -# version_path_separator = space -# version_path_separator = newline -version_path_separator = os # Use os.pathsep. Default configuration used for new projects. - -# set to 'true' to search source files recursively -# in each "version_locations" directory -# new in Alembic version 1.10 -# recursive_version_locations = false - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -sqlalchemy.url = driver://user:pass@localhost/dbname - - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME - -# lint with attempts to fix using "ruff" - use the exec runner, execute a binary -# hooks = ruff -# ruff.type = exec -# ruff.executable = %(here)s/.venv/bin/ruff -# ruff.options = --fix REVISION_SCRIPT_FILENAME - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARNING -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARNING -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/src/models/db_schemes/minirag/alembic/README b/src/models/db_schemes/minirag/alembic/README deleted file mode 100644 index 98e4f9c44..000000000 --- a/src/models/db_schemes/minirag/alembic/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/src/models/db_schemes/minirag/alembic/env.py b/src/models/db_schemes/minirag/alembic/env.py deleted file mode 100644 index 7b1c34626..000000000 --- a/src/models/db_schemes/minirag/alembic/env.py +++ /dev/null @@ -1,79 +0,0 @@ -from logging.config import fileConfig - -from sqlalchemy import engine_from_config -from sqlalchemy import pool -from schemes import SQLAlchemyBase - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = SQLAlchemyBase.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/src/models/db_schemes/minirag/alembic/script.py.mako b/src/models/db_schemes/minirag/alembic/script.py.mako deleted file mode 100644 index fbc4b07dc..000000000 --- a/src/models/db_schemes/minirag/alembic/script.py.mako +++ /dev/null @@ -1,26 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} diff --git a/src/models/db_schemes/minirag/alembic/versions/243ca8b683b0_update_celery_task_executions_table_.py b/src/models/db_schemes/minirag/alembic/versions/243ca8b683b0_update_celery_task_executions_table_.py deleted file mode 100644 index bc504d12e..000000000 --- a/src/models/db_schemes/minirag/alembic/versions/243ca8b683b0_update_celery_task_executions_table_.py +++ /dev/null @@ -1,30 +0,0 @@ -"""update celery_task_executions table indexes - -Revision ID: 243ca8b683b0 -Revises: b9f9e870b09b -Create Date: 2025-08-03 23:15:43.860171 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision: str = '243ca8b683b0' -down_revision: Union[str, None] = 'b9f9e870b09b' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_index('ixz_task_name_args_celery_hash', 'celery_task_executions', ['task_name', 'task_args_hash', 'celery_task_id'], unique=True) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index('ixz_task_name_args_celery_hash', table_name='celery_task_executions') - # ### end Alembic commands ### diff --git a/src/models/db_schemes/minirag/alembic/versions/b9f9e870b09b_create_celery_task_executions_table.py b/src/models/db_schemes/minirag/alembic/versions/b9f9e870b09b_create_celery_task_executions_table.py deleted file mode 100644 index ad406e139..000000000 --- a/src/models/db_schemes/minirag/alembic/versions/b9f9e870b09b_create_celery_task_executions_table.py +++ /dev/null @@ -1,51 +0,0 @@ -"""create celery_task_executions table - -Revision ID: b9f9e870b09b -Revises: fee4cd54bd38 -Create Date: 2025-08-03 22:17:07.184977 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = 'b9f9e870b09b' -down_revision: Union[str, None] = 'fee4cd54bd38' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('celery_task_executions', - sa.Column('execution_id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('task_name', sa.String(length=255), nullable=False), - sa.Column('task_args_hash', sa.String(length=64), nullable=False), - sa.Column('celery_task_id', sa.UUID(), nullable=True), - sa.Column('status', sa.String(length=20), nullable=False), - sa.Column('task_args', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('result', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('execution_id') - ) - op.create_index('ixz_celery_task_id', 'celery_task_executions', ['celery_task_id'], unique=False) - op.create_index('ixz_task_execution_created_at', 'celery_task_executions', ['created_at'], unique=False) - op.create_index('ixz_task_execution_status', 'celery_task_executions', ['status'], unique=False) - op.create_index('ixz_task_name_args_hash', 'celery_task_executions', ['task_name', 'task_args_hash'], unique=True) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index('ixz_task_name_args_hash', table_name='celery_task_executions') - op.drop_index('ixz_task_execution_status', table_name='celery_task_executions') - op.drop_index('ixz_task_execution_created_at', table_name='celery_task_executions') - op.drop_index('ixz_celery_task_id', table_name='celery_task_executions') - op.drop_table('celery_task_executions') - # ### end Alembic commands ### diff --git a/src/models/db_schemes/minirag/alembic/versions/fee4cd54bd38_initial_commit.py b/src/models/db_schemes/minirag/alembic/versions/fee4cd54bd38_initial_commit.py deleted file mode 100644 index e72bda010..000000000 --- a/src/models/db_schemes/minirag/alembic/versions/fee4cd54bd38_initial_commit.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Initial Commit - -Revision ID: fee4cd54bd38 -Revises: -Create Date: 2024-12-02 11:21:07.921865 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision: str = 'fee4cd54bd38' -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('projects', - sa.Column('project_id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('project_uuid', sa.UUID(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('project_id'), - sa.UniqueConstraint('project_uuid') - ) - op.create_table('assets', - sa.Column('asset_id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('asset_uuid', sa.UUID(), nullable=False), - sa.Column('asset_type', sa.String(), nullable=False), - sa.Column('asset_name', sa.String(), nullable=False), - sa.Column('asset_size', sa.Integer(), nullable=False), - sa.Column('asset_config', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('asset_project_id', sa.Integer(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['asset_project_id'], ['projects.project_id'], ), - sa.PrimaryKeyConstraint('asset_id'), - sa.UniqueConstraint('asset_uuid') - ) - op.create_index('ix_asset_project_id', 'assets', ['asset_project_id'], unique=False) - op.create_index('ix_asset_type', 'assets', ['asset_type'], unique=False) - op.create_table('chunks', - sa.Column('chunk_id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('chunk_uuid', sa.UUID(), nullable=False), - sa.Column('chunk_text', sa.String(), nullable=False), - sa.Column('chunk_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('chunk_order', sa.Integer(), nullable=False), - sa.Column('chunk_project_id', sa.Integer(), nullable=False), - sa.Column('chunk_asset_id', sa.Integer(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['chunk_asset_id'], ['assets.asset_id'], ), - sa.ForeignKeyConstraint(['chunk_project_id'], ['projects.project_id'], ), - sa.PrimaryKeyConstraint('chunk_id'), - sa.UniqueConstraint('chunk_uuid') - ) - op.create_index('ix_chunk_asset_id', 'chunks', ['chunk_asset_id'], unique=False) - op.create_index('ix_chunk_project_id', 'chunks', ['chunk_project_id'], unique=False) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index('ix_chunk_project_id', table_name='chunks') - op.drop_index('ix_chunk_asset_id', table_name='chunks') - op.drop_table('chunks') - op.drop_index('ix_asset_type', table_name='assets') - op.drop_index('ix_asset_project_id', table_name='assets') - op.drop_table('assets') - op.drop_table('projects') - # ### end Alembic commands ### diff --git a/src/models/db_schemes/minirag/schemes/__init__.py b/src/models/db_schemes/minirag/schemes/__init__.py deleted file mode 100644 index 478a864d4..000000000 --- a/src/models/db_schemes/minirag/schemes/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .minirag_base import SQLAlchemyBase -from .asset import Asset -from .project import Project -from .datachunk import DataChunk, RetrievedDocument -from .celery_task_execution import CeleryTaskExecution diff --git a/src/models/db_schemes/minirag/schemes/asset.py b/src/models/db_schemes/minirag/schemes/asset.py deleted file mode 100644 index bce853bd6..000000000 --- a/src/models/db_schemes/minirag/schemes/asset.py +++ /dev/null @@ -1,32 +0,0 @@ -from .minirag_base import SQLAlchemyBase -from sqlalchemy import Column, Integer, DateTime, func, String, ForeignKey -from sqlalchemy.dialects.postgresql import UUID, JSONB -from sqlalchemy.orm import relationship -from sqlalchemy import Index -import uuid - -class Asset(SQLAlchemyBase): - - __tablename__ = "assets" - - asset_id = Column(Integer, primary_key=True, autoincrement=True) - asset_uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, unique=True, nullable=False) - - asset_type = Column(String, nullable=False) - asset_name = Column(String, nullable=False) - asset_size = Column(Integer, nullable=False) - asset_config = Column(JSONB, nullable=True) - - asset_project_id = Column(Integer, ForeignKey("projects.project_id"), nullable=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - updated_at = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) - - project = relationship("Project", back_populates="assets") - chunks = relationship("DataChunk", back_populates="asset") - - __table_args__ = ( - Index('ix_asset_project_id', asset_project_id), - Index('ix_asset_type', asset_type), - ) - diff --git a/src/models/db_schemes/minirag/schemes/celery_task_execution.py b/src/models/db_schemes/minirag/schemes/celery_task_execution.py deleted file mode 100644 index 8ef63aa6c..000000000 --- a/src/models/db_schemes/minirag/schemes/celery_task_execution.py +++ /dev/null @@ -1,34 +0,0 @@ -from .minirag_base import SQLAlchemyBase -from sqlalchemy import Column, Integer, DateTime, func, String, Text -from sqlalchemy.dialects.postgresql import UUID, JSONB -from sqlalchemy import Index -import uuid - -class CeleryTaskExecution(SQLAlchemyBase): - - __tablename__ = "celery_task_executions" - - execution_id = Column(Integer, primary_key=True, autoincrement=True) - - task_name = Column(String(255), nullable=False) - task_args_hash = Column(String(64), nullable=False) # SHA-256 hash of task arguments - celery_task_id = Column(UUID(as_uuid=True), nullable=True) - - status = Column(String(20), nullable=False, default='PENDING') - - task_args = Column(JSONB, nullable=True) - result = Column(JSONB, nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - completed_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - updated_at = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) - - __table_args__ = ( - Index('ixz_task_name_args_celery_hash', task_name, task_args_hash, celery_task_id, unique=True), - Index('ixz_task_execution_status', status), - Index('ixz_task_execution_created_at', created_at), - Index('ixz_celery_task_id', celery_task_id), - ) - - diff --git a/src/models/db_schemes/minirag/schemes/datachunk.py b/src/models/db_schemes/minirag/schemes/datachunk.py deleted file mode 100644 index 536ca1b27..000000000 --- a/src/models/db_schemes/minirag/schemes/datachunk.py +++ /dev/null @@ -1,36 +0,0 @@ -from .minirag_base import SQLAlchemyBase -from sqlalchemy import Column, Integer, DateTime, func, String, ForeignKey -from sqlalchemy.dialects.postgresql import UUID, JSONB -from sqlalchemy.orm import relationship -from sqlalchemy import Index -from pydantic import BaseModel -import uuid - -class DataChunk(SQLAlchemyBase): - - __tablename__ = "chunks" - - chunk_id = Column(Integer, primary_key=True, autoincrement=True) - chunk_uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, unique=True, nullable=False) - - chunk_text = Column(String, nullable=False) - chunk_metadata = Column(JSONB, nullable=True) - chunk_order = Column(Integer, nullable=False) - - chunk_project_id = Column(Integer, ForeignKey("projects.project_id"), nullable=False) - chunk_asset_id = Column(Integer, ForeignKey("assets.asset_id"), nullable=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - updated_at = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) - - project = relationship("Project", back_populates="chunks") - asset = relationship("Asset", back_populates="chunks") - - __table_args__ = ( - Index('ix_chunk_project_id', chunk_project_id), - Index('ix_chunk_asset_id', chunk_asset_id), - ) - -class RetrievedDocument(BaseModel): - text: str - score: float diff --git a/src/models/db_schemes/minirag/schemes/minirag_base.py b/src/models/db_schemes/minirag/schemes/minirag_base.py deleted file mode 100644 index e40835fe8..000000000 --- a/src/models/db_schemes/minirag/schemes/minirag_base.py +++ /dev/null @@ -1,2 +0,0 @@ -from sqlalchemy.ext.declarative import declarative_base -SQLAlchemyBase = declarative_base() diff --git a/src/models/db_schemes/minirag/schemes/project.py b/src/models/db_schemes/minirag/schemes/project.py deleted file mode 100644 index c41dcf00b..000000000 --- a/src/models/db_schemes/minirag/schemes/project.py +++ /dev/null @@ -1,18 +0,0 @@ -from .minirag_base import SQLAlchemyBase -from sqlalchemy import Column, Integer, DateTime, func -from sqlalchemy.dialects.postgresql import UUID -import uuid -from sqlalchemy.orm import relationship - -class Project(SQLAlchemyBase): - - __tablename__ = "projects" - - project_id = Column(Integer, primary_key=True, autoincrement=True) - project_uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, unique=True, nullable=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - updated_at = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) - - chunks = relationship("DataChunk", back_populates="project") - assets = relationship("Asset", back_populates="project") diff --git a/src/models/enums/AssetTypeEnum.py b/src/models/enums/AssetTypeEnum.py deleted file mode 100644 index 0e849ae82..000000000 --- a/src/models/enums/AssetTypeEnum.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import Enum - -class AssetTypeEnum(Enum): - - FILE = "file" - \ No newline at end of file diff --git a/src/models/enums/DataBaseEnum.py b/src/models/enums/DataBaseEnum.py deleted file mode 100644 index bb11bf715..000000000 --- a/src/models/enums/DataBaseEnum.py +++ /dev/null @@ -1,8 +0,0 @@ -from enum import Enum - -class DataBaseEnum(Enum): - - COLLECTION_PROJECT_NAME = "projects" - COLLECTION_CHUNK_NAME = "chunks" - COLLECTION_ASSET_NAME = "assets" - diff --git a/src/models/enums/ProcessingEnum.py b/src/models/enums/ProcessingEnum.py deleted file mode 100644 index f6f4c3a7a..000000000 --- a/src/models/enums/ProcessingEnum.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import Enum - -class ProcessingEnum(Enum): - - TXT = ".txt" - PDF = ".pdf" diff --git a/src/models/enums/ResponseEnums.py b/src/models/enums/ResponseEnums.py deleted file mode 100644 index aa46a0929..000000000 --- a/src/models/enums/ResponseEnums.py +++ /dev/null @@ -1,24 +0,0 @@ -from enum import Enum - -class ResponseSignal(Enum): - - FILE_VALIDATED_SUCCESS = "file_validate_successfully" - FILE_TYPE_NOT_SUPPORTED = "file_type_not_supported" - FILE_SIZE_EXCEEDED = "file_size_exceeded" - FILE_UPLOAD_SUCCESS = "file_upload_success" - FILE_UPLOAD_FAILED = "file_upload_failed" - PROCESSING_SUCCESS = "processing_success" - PROCESSING_FAILED = "processing_failed" - NO_FILES_ERROR = "not_found_files" - FILE_ID_ERROR = "no_file_found_with_this_id" - PROJECT_NOT_FOUND_ERROR = "project_not_found" - INSERT_INTO_VECTORDB_ERROR = "insert_into_vectordb_error" - INSERT_INTO_VECTORDB_SUCCESS = "insert_into_vectordb_success" - VECTORDB_COLLECTION_RETRIEVED = "vectordb_collection_retrieved" - VECTORDB_SEARCH_ERROR = "vectordb_search_error" - VECTORDB_SEARCH_SUCCESS = "vectordb_search_success" - RAG_ANSWER_ERROR = "rag_answer_error" - RAG_ANSWER_SUCCESS = "rag_answer_success" - DATA_PUSH_TASK_READY="data_push_task_ready" - PROCESS_AND_PUSH_WORKFLOW_READY="process_and_push_workflow_ready" - \ No newline at end of file diff --git a/src/models/enums/__init__.py b/src/models/enums/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/requirements.txt b/src/requirements.txt deleted file mode 100644 index cb2b5a905..000000000 --- a/src/requirements.txt +++ /dev/null @@ -1,32 +0,0 @@ -fastapi==0.110.2 -uvicorn[standard]==0.29.0 -python-multipart==0.0.9 -python-dotenv==1.0.1 -pydantic-settings==2.2.1 -aiofiles==23.2.1 -langchain==0.1.20 -PyMuPDF==1.24.3 -motor==3.4.0 -pydantic-mongo==2.3.0 -openai==1.75.0 -cohere==5.5.8 -qdrant-client==1.10.1 -SQLAlchemy==2.0.36 -asyncpg==0.30.0 -alembic==1.14.0 -psycopg2==2.9.10 -pgvector==0.4.0 -nltk==3.9.1 - -# Monitoring and metrics -prometheus-client==0.21.1 -starlette-exporter==0.23.0 -fastapi-health==0.4.0 - -# Task Queue and Background Processing -celery==5.5.3 -redis==6.2.0 -kombu==5.5.4 -billiard==4.2.1 -vine==5.1.0 -flower==2.0.1 \ No newline at end of file diff --git a/src/routes/__init__.py b/src/routes/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/routes/base.py b/src/routes/base.py deleted file mode 100644 index 45d009e95..000000000 --- a/src/routes/base.py +++ /dev/null @@ -1,23 +0,0 @@ -from fastapi import FastAPI, APIRouter, Depends -import os -from helpers.config import get_settings, Settings -from time import sleep -import logging - -logger = logging.getLogger('uvicorn.error') - -base_router = APIRouter( - prefix="/api/v1", - tags=["api_v1"], -) - -@base_router.get("/") -async def welcome(app_settings: Settings = Depends(get_settings)): - - app_name = app_settings.APP_NAME - app_version = app_settings.APP_VERSION - - return { - "app_name": app_name, - "app_version": app_version, - } diff --git a/src/routes/data.py b/src/routes/data.py deleted file mode 100644 index b419c6882..000000000 --- a/src/routes/data.py +++ /dev/null @@ -1,136 +0,0 @@ -from fastapi import FastAPI, APIRouter, Depends, UploadFile, status, Request -from fastapi.responses import JSONResponse -import os -from helpers.config import get_settings, Settings -from controllers import DataController, ProjectController, ProcessController -import aiofiles -from models import ResponseSignal -import logging -from .schemes.data import ProcessRequest -from models.ProjectModel import ProjectModel -from models.ChunkModel import ChunkModel -from models.AssetModel import AssetModel -from models.db_schemes import DataChunk, Asset -from models.enums.AssetTypeEnum import AssetTypeEnum -from controllers import NLPController -from tasks.file_processing import process_project_files -from tasks.process_workflow import process_and_push_workflow - -logger = logging.getLogger('uvicorn.error') - -data_router = APIRouter( - prefix="/api/v1/data", - tags=["api_v1", "data"], -) - -@data_router.post("/upload/{project_id}") -async def upload_data(request: Request, project_id: int, file: UploadFile, - app_settings: Settings = Depends(get_settings)): - - - project_model = await ProjectModel.create_instance( - db_client=request.app.db_client - ) - - project = await project_model.get_project_or_create_one( - project_id=project_id - ) - - # validate the file properties - data_controller = DataController() - - is_valid, result_signal = data_controller.validate_uploaded_file(file=file) - - if not is_valid: - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={ - "signal": result_signal - } - ) - - project_dir_path = ProjectController().get_project_path(project_id=project_id) - file_path, file_id = data_controller.generate_unique_filepath( - orig_file_name=file.filename, - project_id=project_id - ) - - try: - async with aiofiles.open(file_path, "wb") as f: - while chunk := await file.read(app_settings.FILE_DEFAULT_CHUNK_SIZE): - await f.write(chunk) - except Exception as e: - - logger.error(f"Error while uploading file: {e}") - - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={ - "signal": ResponseSignal.FILE_UPLOAD_FAILED.value - } - ) - - # store the assets into the database - asset_model = await AssetModel.create_instance( - db_client=request.app.db_client - ) - - asset_resource = Asset( - asset_project_id=project.project_id, - asset_type=AssetTypeEnum.FILE.value, - asset_name=file_id, - asset_size=os.path.getsize(file_path) - ) - - asset_record = await asset_model.create_asset(asset=asset_resource) - - return JSONResponse( - content={ - "signal": ResponseSignal.FILE_UPLOAD_SUCCESS.value, - "file_id": str(asset_record.asset_id), - } - ) - -@data_router.post("/process/{project_id}") -async def process_endpoint(request: Request, project_id: int, process_request: ProcessRequest): - - chunk_size = process_request.chunk_size - overlap_size = process_request.overlap_size - do_reset = process_request.do_reset - - task = process_project_files.delay( - project_id=project_id, - file_id=process_request.file_id, - chunk_size=chunk_size, - overlap_size=overlap_size, - do_reset=do_reset, - ) - - return JSONResponse( - content={ - "signal": ResponseSignal.PROCESSING_SUCCESS.value, - "task_id": task.id - } - ) - -@data_router.post("/process-and-push/{project_id}") -async def process_and_push_endpoint(request: Request, project_id: int, process_request: ProcessRequest): - - chunk_size = process_request.chunk_size - overlap_size = process_request.overlap_size - do_reset = process_request.do_reset - - workflow_task = process_and_push_workflow.delay( - project_id=project_id, - file_id=process_request.file_id, - chunk_size=chunk_size, - overlap_size=overlap_size, - do_reset=do_reset, - ) - - return JSONResponse( - content={ - "signal": ResponseSignal.PROCESS_AND_PUSH_WORKFLOW_READY.value, - "workflow_task_id": workflow_task.id - } - ) diff --git a/src/routes/nlp.py b/src/routes/nlp.py deleted file mode 100644 index ccd273e7d..000000000 --- a/src/routes/nlp.py +++ /dev/null @@ -1,139 +0,0 @@ -from fastapi import FastAPI, APIRouter, status, Request -from fastapi.responses import JSONResponse -from routes.schemes.nlp import PushRequest, SearchRequest -from models.ProjectModel import ProjectModel -from models.ChunkModel import ChunkModel -from controllers import NLPController -from models import ResponseSignal -from tqdm.auto import tqdm -from tasks.data_indexing import index_data_content - -import logging - -logger = logging.getLogger('uvicorn.error') - -nlp_router = APIRouter( - prefix="/api/v1/nlp", - tags=["api_v1", "nlp"], -) - -@nlp_router.post("/index/push/{project_id}") -async def index_project(request: Request, project_id: int, push_request: PushRequest): - - task = index_data_content.delay( - project_id=project_id, - do_reset=push_request.do_reset - ) - - return JSONResponse( - content={ - "signal": ResponseSignal.DATA_PUSH_TASK_READY.value, - "task_id": task.id - } - ) - - -@nlp_router.get("/index/info/{project_id}") -async def get_project_index_info(request: Request, project_id: int): - - project_model = await ProjectModel.create_instance( - db_client=request.app.db_client - ) - - project = await project_model.get_project_or_create_one( - project_id=project_id - ) - - nlp_controller = NLPController( - vectordb_client=request.app.vectordb_client, - generation_client=request.app.generation_client, - embedding_client=request.app.embedding_client, - template_parser=request.app.template_parser, - ) - - collection_info = await nlp_controller.get_vector_db_collection_info(project=project) - - return JSONResponse( - content={ - "signal": ResponseSignal.VECTORDB_COLLECTION_RETRIEVED.value, - "collection_info": collection_info - } - ) - -@nlp_router.post("/index/search/{project_id}") -async def search_index(request: Request, project_id: int, search_request: SearchRequest): - - project_model = await ProjectModel.create_instance( - db_client=request.app.db_client - ) - - project = await project_model.get_project_or_create_one( - project_id=project_id - ) - - nlp_controller = NLPController( - vectordb_client=request.app.vectordb_client, - generation_client=request.app.generation_client, - embedding_client=request.app.embedding_client, - template_parser=request.app.template_parser, - ) - - results = await nlp_controller.search_vector_db_collection( - project=project, text=search_request.text, limit=search_request.limit - ) - - if not results: - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={ - "signal": ResponseSignal.VECTORDB_SEARCH_ERROR.value - } - ) - - return JSONResponse( - content={ - "signal": ResponseSignal.VECTORDB_SEARCH_SUCCESS.value, - "results": [ result.dict() for result in results ] - } - ) - -@nlp_router.post("/index/answer/{project_id}") -async def answer_rag(request: Request, project_id: int, search_request: SearchRequest): - - project_model = await ProjectModel.create_instance( - db_client=request.app.db_client - ) - - project = await project_model.get_project_or_create_one( - project_id=project_id - ) - - nlp_controller = NLPController( - vectordb_client=request.app.vectordb_client, - generation_client=request.app.generation_client, - embedding_client=request.app.embedding_client, - template_parser=request.app.template_parser, - ) - - answer, full_prompt, chat_history = await nlp_controller.answer_rag_question( - project=project, - query=search_request.text, - limit=search_request.limit, - ) - - if not answer: - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content={ - "signal": ResponseSignal.RAG_ANSWER_ERROR.value - } - ) - - return JSONResponse( - content={ - "signal": ResponseSignal.RAG_ANSWER_SUCCESS.value, - "answer": answer, - "full_prompt": full_prompt, - "chat_history": chat_history - } - ) diff --git a/src/routes/schemes/__init__.py b/src/routes/schemes/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/routes/schemes/data.py b/src/routes/schemes/data.py deleted file mode 100644 index 2d72068b2..000000000 --- a/src/routes/schemes/data.py +++ /dev/null @@ -1,8 +0,0 @@ -from pydantic import BaseModel -from typing import Optional - -class ProcessRequest(BaseModel): - file_id: str = None - chunk_size: Optional[int] = 100 - overlap_size: Optional[int] = 20 - do_reset: Optional[int] = 0 diff --git a/src/routes/schemes/nlp.py b/src/routes/schemes/nlp.py deleted file mode 100644 index 573194847..000000000 --- a/src/routes/schemes/nlp.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic import BaseModel -from typing import Optional - -class PushRequest(BaseModel): - do_reset: Optional[int] = 0 - -class SearchRequest(BaseModel): - text: str - limit: Optional[int] = 5 diff --git a/src/stores/llm/LLMEnums.py b/src/stores/llm/LLMEnums.py deleted file mode 100644 index bd7c8cfcb..000000000 --- a/src/stores/llm/LLMEnums.py +++ /dev/null @@ -1,23 +0,0 @@ -from enum import Enum - -class LLMEnums(Enum): - OPENAI = "OPENAI" - COHERE = "COHERE" - -class OpenAIEnums(Enum): - SYSTEM = "system" - USER = "user" - ASSISTANT = "assistant" - -class CoHereEnums(Enum): - SYSTEM = "SYSTEM" - USER = "USER" - ASSISTANT = "CHATBOT" - - DOCUMENT = "search_document" - QUERY = "search_query" - - -class DocumentTypeEnum(Enum): - DOCUMENT = "document" - QUERY = "query" \ No newline at end of file diff --git a/src/stores/llm/LLMInterface.py b/src/stores/llm/LLMInterface.py deleted file mode 100644 index a86ebde59..000000000 --- a/src/stores/llm/LLMInterface.py +++ /dev/null @@ -1,24 +0,0 @@ -from abc import ABC, abstractmethod - -class LLMInterface(ABC): - - @abstractmethod - def set_generation_model(self, model_id: str): - pass - - @abstractmethod - def set_embedding_model(self, model_id: str, embedding_size: int): - pass - - @abstractmethod - def generate_text(self, prompt: str, chat_history: list=[], max_output_tokens: int=None, - temperature: float = None): - pass - - @abstractmethod - def embed_text(self, text: str, document_type: str = None): - pass - - @abstractmethod - def construct_prompt(self, prompt: str, role: str): - pass diff --git a/src/stores/llm/LLMProviderFactory.py b/src/stores/llm/LLMProviderFactory.py deleted file mode 100644 index 2cd253929..000000000 --- a/src/stores/llm/LLMProviderFactory.py +++ /dev/null @@ -1,27 +0,0 @@ - -from .LLMEnums import LLMEnums -from .providers import OpenAIProvider, CoHereProvider - -class LLMProviderFactory: - def __init__(self, config: dict): - self.config = config - - def create(self, provider: str): - if provider == LLMEnums.OPENAI.value: - return OpenAIProvider( - api_key = self.config.OPENAI_API_KEY, - api_url = self.config.OPENAI_API_URL, - default_input_max_characters=self.config.INPUT_DAFAULT_MAX_CHARACTERS, - default_generation_max_output_tokens=self.config.GENERATION_DAFAULT_MAX_TOKENS, - default_generation_temperature=self.config.GENERATION_DAFAULT_TEMPERATURE - ) - - if provider == LLMEnums.COHERE.value: - return CoHereProvider( - api_key = self.config.COHERE_API_KEY, - default_input_max_characters=self.config.INPUT_DAFAULT_MAX_CHARACTERS, - default_generation_max_output_tokens=self.config.GENERATION_DAFAULT_MAX_TOKENS, - default_generation_temperature=self.config.GENERATION_DAFAULT_TEMPERATURE - ) - - return None diff --git a/src/stores/llm/__init__.py b/src/stores/llm/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/stores/llm/providers/CoHereProvider.py b/src/stores/llm/providers/CoHereProvider.py deleted file mode 100644 index ac0cac4f7..000000000 --- a/src/stores/llm/providers/CoHereProvider.py +++ /dev/null @@ -1,101 +0,0 @@ -from ..LLMInterface import LLMInterface -from ..LLMEnums import CoHereEnums, DocumentTypeEnum -import cohere -import logging -from typing import List, Union - -class CoHereProvider(LLMInterface): - - def __init__(self, api_key: str, - default_input_max_characters: int=1000, - default_generation_max_output_tokens: int=1000, - default_generation_temperature: float=0.1): - - self.api_key = api_key - - self.default_input_max_characters = default_input_max_characters - self.default_generation_max_output_tokens = default_generation_max_output_tokens - self.default_generation_temperature = default_generation_temperature - - self.generation_model_id = None - - self.embedding_model_id = None - self.embedding_size = None - - self.client = cohere.Client(api_key=self.api_key) - - self.enums = CoHereEnums - self.logger = logging.getLogger(__name__) - - def set_generation_model(self, model_id: str): - self.generation_model_id = model_id - - def set_embedding_model(self, model_id: str, embedding_size: int): - self.embedding_model_id = model_id - self.embedding_size = embedding_size - - def process_text(self, text: str): - return text[:self.default_input_max_characters].strip() - - def generate_text(self, prompt: str, chat_history: list=[], max_output_tokens: int=None, - temperature: float = None): - - if not self.client: - self.logger.error("CoHere client was not set") - return None - - if not self.generation_model_id: - self.logger.error("Generation model for CoHere was not set") - return None - - max_output_tokens = max_output_tokens if max_output_tokens else self.default_generation_max_output_tokens - temperature = temperature if temperature else self.default_generation_temperature - - response = self.client.chat( - model = self.generation_model_id, - chat_history = chat_history, - message = self.process_text(prompt), - temperature = temperature, - max_tokens = max_output_tokens - ) - - if not response or not response.text: - self.logger.error("Error while generating text with CoHere") - return None - - return response.text - - def embed_text(self, text: Union[str, List[str]], document_type: str = None): - if not self.client: - self.logger.error("CoHere client was not set") - return None - - if isinstance(text, str): - text = [text] - - if not self.embedding_model_id: - self.logger.error("Embedding model for CoHere was not set") - return None - - input_type = CoHereEnums.DOCUMENT - if document_type == DocumentTypeEnum.QUERY: - input_type = CoHereEnums.QUERY - - response = self.client.embed( - model = self.embedding_model_id, - texts = [ self.process_text(t) for t in text ], - input_type = input_type, - embedding_types=['float'], - ) - - if not response or not response.embeddings or not response.embeddings.float: - self.logger.error("Error while embedding text with CoHere") - return None - - return [ f for f in response.embeddings.float ] - - def construct_prompt(self, prompt: str, role: str): - return { - "role": role, - "text": prompt, - } \ No newline at end of file diff --git a/src/stores/llm/providers/OpenAIProvider.py b/src/stores/llm/providers/OpenAIProvider.py deleted file mode 100644 index c1a2b375d..000000000 --- a/src/stores/llm/providers/OpenAIProvider.py +++ /dev/null @@ -1,109 +0,0 @@ -from ..LLMInterface import LLMInterface -from ..LLMEnums import OpenAIEnums -from openai import OpenAI -import logging -from typing import List, Union - -class OpenAIProvider(LLMInterface): - - def __init__(self, api_key: str, api_url: str=None, - default_input_max_characters: int=1000, - default_generation_max_output_tokens: int=1000, - default_generation_temperature: float=0.1): - - self.api_key = api_key - self.api_url = api_url - - self.default_input_max_characters = default_input_max_characters - self.default_generation_max_output_tokens = default_generation_max_output_tokens - self.default_generation_temperature = default_generation_temperature - - self.generation_model_id = None - - self.embedding_model_id = None - self.embedding_size = None - - self.client = OpenAI( - api_key = self.api_key, - base_url = self.api_url if self.api_url and len(self.api_url) else None - ) - - self.enums = OpenAIEnums - self.logger = logging.getLogger(__name__) - - def set_generation_model(self, model_id: str): - self.generation_model_id = model_id - - def set_embedding_model(self, model_id: str, embedding_size: int): - self.embedding_model_id = model_id - self.embedding_size = embedding_size - - def process_text(self, text: str): - return text[:self.default_input_max_characters].strip() - - def generate_text(self, prompt: str, chat_history: list=[], max_output_tokens: int=None, - temperature: float = None): - - if not self.client: - self.logger.error("OpenAI client was not set") - return None - - if not self.generation_model_id: - self.logger.error("Generation model for OpenAI was not set") - return None - - max_output_tokens = max_output_tokens if max_output_tokens else self.default_generation_max_output_tokens - temperature = temperature if temperature else self.default_generation_temperature - - chat_history.append( - self.construct_prompt(prompt=prompt, role=OpenAIEnums.USER.value) - ) - - response = self.client.chat.completions.create( - model = self.generation_model_id, - messages = chat_history, - max_tokens = max_output_tokens, - temperature = temperature - ) - - if not response or not response.choices or len(response.choices) == 0 or not response.choices[0].message: - self.logger.error("Error while generating text with OpenAI") - return None - - return response.choices[0].message.content - - - def embed_text(self, text: Union[str, List[str]], document_type: str = None): - - if not self.client: - self.logger.error("OpenAI client was not set") - return None - - if isinstance(text, str): - text = [text] - - if not self.embedding_model_id: - self.logger.error("Embedding model for OpenAI was not set") - return None - - response = self.client.embeddings.create( - model = self.embedding_model_id, - input = text, - ) - - if not response or not response.data or len(response.data) == 0 or not response.data[0].embedding: - self.logger.error("Error while embedding text with OpenAI") - return None - - return [ rec.embedding for rec in response.data ] - - def construct_prompt(self, prompt: str, role: str): - return { - "role": role, - "content": prompt, - } - - - - - diff --git a/src/stores/llm/providers/__init__.py b/src/stores/llm/providers/__init__.py deleted file mode 100644 index 3368d7662..000000000 --- a/src/stores/llm/providers/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .CoHereProvider import CoHereProvider -from .OpenAIProvider import OpenAIProvider diff --git a/src/stores/llm/templates/__init__.py b/src/stores/llm/templates/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/stores/llm/templates/locales/__init__.py b/src/stores/llm/templates/locales/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/stores/llm/templates/locales/ar/__init__.py b/src/stores/llm/templates/locales/ar/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/stores/llm/templates/locales/ar/rag.py b/src/stores/llm/templates/locales/ar/rag.py deleted file mode 100644 index 80c188178..000000000 --- a/src/stores/llm/templates/locales/ar/rag.py +++ /dev/null @@ -1,33 +0,0 @@ -from string import Template - -#### RAG PROMPTS #### - -#### System #### - -system_prompt = Template("\n".join([ - "أنت مساعد لتوليد رد للمستخدم.", - "ستحصل على مجموعة من المستندات المرتبطة باستفسار المستخدم.", - "عليك توليد رد بناءً على المستندات المقدمة.", - "تجاهل المستندات التي لا تتعلق باستفسار المستخدم.", - "يمكنك الاعتذار للمستخدم إذا لم تتمكن من توليد رد.", - "عليك توليد الرد بنفس لغة استفسار المستخدم.", - "كن مؤدباً ومحترماً في التعامل مع المستخدم.", - "كن دقيقًا ومختصرًا في ردك. تجنب المعلومات غير الضرورية.", -])) - -#### Document #### -document_prompt = Template( - "\n".join([ - "## المستند رقم: $doc_num", - "### المحتوى: $chunk_text", - ]) -) - -#### Footer #### -footer_prompt = Template("\n".join([ - "بناءً فقط على المستندات المذكورة أعلاه، يرجى توليد إجابة للمستخدم.", - "## السؤال:", - "$query", - "", - "## الإجابة:", -])) \ No newline at end of file diff --git a/src/stores/llm/templates/locales/en/__init__.py b/src/stores/llm/templates/locales/en/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/stores/llm/templates/locales/en/rag.py b/src/stores/llm/templates/locales/en/rag.py deleted file mode 100644 index f784e349e..000000000 --- a/src/stores/llm/templates/locales/en/rag.py +++ /dev/null @@ -1,33 +0,0 @@ -from string import Template - -#### RAG PROMPTS #### - -#### System #### - -system_prompt = Template("\n".join([ - "You are an assistant to generate a response for the user.", - "You will be provided by a set of docuemnts associated with the user's query.", - "You have to generate a response based on the documents provided.", - "Ignore the documents that are not relevant to the user's query.", - "You can applogize to the user if you are not able to generate a response.", - "You have to generate response in the same language as the user's query.", - "Be polite and respectful to the user.", - "Be precise and concise in your response. Avoid unnecessary information.", -])) - -#### Document #### -document_prompt = Template( - "\n".join([ - "## Document No: $doc_num", - "### Content: $chunk_text", - ]) -) - -#### Footer #### -footer_prompt = Template("\n".join([ - "Based only on the above documents, please generate an answer for the user.", - "## Question:", - "$query", - "", - "## Answer:", -])) \ No newline at end of file diff --git a/src/stores/llm/templates/template_parser.py b/src/stores/llm/templates/template_parser.py deleted file mode 100644 index 0cee58eeb..000000000 --- a/src/stores/llm/templates/template_parser.py +++ /dev/null @@ -1,43 +0,0 @@ -import os - -class TemplateParser: - - def __init__(self, language: str=None, default_language='en'): - self.current_path = os.path.dirname(os.path.abspath(__file__)) - self.default_language = default_language - self.language = None - - self.set_language(language) - - - def set_language(self, language: str): - if not language: - self.language = self.default_language - - language_path = os.path.join(self.current_path, "locales", language) - if os.path.exists(language_path): - self.language = language - else: - self.language = self.default_language - - def get(self, group: str, key: str, vars: dict={}): - if not group or not key: - return None - - group_path = os.path.join(self.current_path, "locales", self.language, f"{group}.py" ) - targeted_language = self.language - if not os.path.exists(group_path): - group_path = os.path.join(self.current_path, "locales", self.default_language, f"{group}.py" ) - targeted_language = self.default_language - - if not os.path.exists(group_path): - return None - - # import group module - module = __import__(f"stores.llm.templates.locales.{targeted_language}.{group}", fromlist=[group]) - - if not module: - return None - - key_attribute = getattr(module, key) - return key_attribute.substitute(vars) diff --git a/src/stores/vectordb/VectorDBEnums.py b/src/stores/vectordb/VectorDBEnums.py deleted file mode 100644 index 783e4d99c..000000000 --- a/src/stores/vectordb/VectorDBEnums.py +++ /dev/null @@ -1,25 +0,0 @@ -from enum import Enum - -class VectorDBEnums(Enum): - QDRANT = "QDRANT" - PGVECTOR = "PGVECTOR" - -class DistanceMethodEnums(Enum): - COSINE = "cosine" - DOT = "dot" - -class PgVectorTableSchemeEnums(Enum): - ID = 'id' - TEXT = 'text' - VECTOR = 'vector' - CHUNK_ID = 'chunk_id' - METADATA = 'metadata' - _PREFIX = 'pgvector' - -class PgVectorDistanceMethodEnums(Enum): - COSINE = "vector_cosine_ops" - DOT = "vector_l2_ops" - -class PgVectorIndexTypeEnums(Enum): - HNSW = "hnsw" - IVFFLAT = "ivfflat" diff --git a/src/stores/vectordb/VectorDBInterface.py b/src/stores/vectordb/VectorDBInterface.py deleted file mode 100644 index 19eb4d9de..000000000 --- a/src/stores/vectordb/VectorDBInterface.py +++ /dev/null @@ -1,52 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List -from models.db_schemes import RetrievedDocument - -class VectorDBInterface(ABC): - - @abstractmethod - def connect(self): - pass - - @abstractmethod - def disconnect(self): - pass - - @abstractmethod - def is_collection_existed(self, collection_name: str) -> bool: - pass - - @abstractmethod - def list_all_collections(self) -> List: - pass - - @abstractmethod - def get_collection_info(self, collection_name: str) -> dict: - pass - - @abstractmethod - def delete_collection(self, collection_name: str): - pass - - @abstractmethod - def create_collection(self, collection_name: str, - embedding_size: int, - do_reset: bool = False): - pass - - @abstractmethod - def insert_one(self, collection_name: str, text: str, vector: list, - metadata: dict = None, - record_id: str = None): - pass - - @abstractmethod - def insert_many(self, collection_name: str, texts: list, - vectors: list, metadata: list = None, - record_ids: list = None, batch_size: int = 50): - pass - - @abstractmethod - def search_by_vector(self, collection_name: str, vector: list, limit: int) -> List[RetrievedDocument]: - pass - \ No newline at end of file diff --git a/src/stores/vectordb/VectorDBProviderFactory.py b/src/stores/vectordb/VectorDBProviderFactory.py deleted file mode 100644 index 0705dff7a..000000000 --- a/src/stores/vectordb/VectorDBProviderFactory.py +++ /dev/null @@ -1,31 +0,0 @@ -from .providers import QdrantDBProvider, PGVectorProvider -from .VectorDBEnums import VectorDBEnums -from controllers.BaseController import BaseController -from sqlalchemy.orm import sessionmaker - -class VectorDBProviderFactory: - def __init__(self, config, db_client: sessionmaker=None): - self.config = config - self.base_controller = BaseController() - self.db_client = db_client - - def create(self, provider: str): - if provider == VectorDBEnums.QDRANT.value: - qdrant_db_client = self.base_controller.get_database_path(db_name=self.config.VECTOR_DB_PATH) - - return QdrantDBProvider( - db_client=qdrant_db_client, - distance_method=self.config.VECTOR_DB_DISTANCE_METHOD, - default_vector_size=self.config.EMBEDDING_MODEL_SIZE, - index_threshold=self.config.VECTOR_DB_PGVEC_INDEX_THRESHOLD, - ) - - if provider == VectorDBEnums.PGVECTOR.value: - return PGVectorProvider( - db_client=self.db_client, - distance_method=self.config.VECTOR_DB_DISTANCE_METHOD, - default_vector_size=self.config.EMBEDDING_MODEL_SIZE, - index_threshold=self.config.VECTOR_DB_PGVEC_INDEX_THRESHOLD, - ) - - return None diff --git a/src/stores/vectordb/__init__.py b/src/stores/vectordb/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/stores/vectordb/providers/PGVectorProvider.py b/src/stores/vectordb/providers/PGVectorProvider.py deleted file mode 100644 index 137a30469..000000000 --- a/src/stores/vectordb/providers/PGVectorProvider.py +++ /dev/null @@ -1,306 +0,0 @@ -from ..VectorDBInterface import VectorDBInterface -from ..VectorDBEnums import (DistanceMethodEnums, PgVectorTableSchemeEnums, - PgVectorDistanceMethodEnums, PgVectorIndexTypeEnums) -import logging -from typing import List -from models.db_schemes import RetrievedDocument -from sqlalchemy.sql import text as sql_text -import json - -class PGVectorProvider(VectorDBInterface): - - def __init__(self, db_client, default_vector_size: int = 786, - distance_method: str = None, index_threshold: int=100): - - self.db_client = db_client - self.default_vector_size = default_vector_size - - self.index_threshold = index_threshold - - if distance_method == DistanceMethodEnums.COSINE.value: - distance_method = PgVectorDistanceMethodEnums.COSINE.value - elif distance_method == DistanceMethodEnums.DOT.value: - distance_method = PgVectorDistanceMethodEnums.DOT.value - - self.pgvector_table_prefix = PgVectorTableSchemeEnums._PREFIX.value - self.distance_method = distance_method - - self.logger = logging.getLogger("uvicorn") - self.default_index_name = lambda collection_name: f"{collection_name}_vector_idx" - - - async def connect(self): - async with self.db_client() as session: - try: - # Check if vector extension already exists - result = await session.execute(sql_text( - "SELECT 1 FROM pg_extension WHERE extname = 'vector'" - )) - extension_exists = result.scalar_one_or_none() - - if not extension_exists: - # Only create if it doesn't exist - await session.execute(sql_text("CREATE EXTENSION vector")) - await session.commit() - except Exception as e: - # If extension already exists or any other error, just log and continue - self.logger.warning(f"Vector extension setup: {str(e)}") - await session.rollback() - - async def disconnect(self): - pass - - async def is_collection_existed(self, collection_name: str) -> bool: - - record = None - async with self.db_client() as session: - async with session.begin(): - list_tbl = sql_text(f'SELECT * FROM pg_tables WHERE tablename = :collection_name') - results = await session.execute(list_tbl, {"collection_name": collection_name}) - record = results.scalar_one_or_none() - - return record - - async def list_all_collections(self) -> List: - records = [] - async with self.db_client() as session: - async with session.begin(): - list_tbl = sql_text('SELECT tablename FROM pg_tables WHERE tablename LIKE :prefix') - results = await session.execute(list_tbl, {"prefix": self.pgvector_table_prefix}) - records = results.scalars().all() - - return records - - async def get_collection_info(self, collection_name: str) -> dict: - async with self.db_client() as session: - async with session.begin(): - - table_info_sql = sql_text(f''' - SELECT schemaname, tablename, tableowner, tablespace, hasindexes - FROM pg_tables - WHERE tablename = :collection_name - ''') - - count_sql = sql_text(f'SELECT COUNT(*) FROM {collection_name}') - - table_info = await session.execute(table_info_sql, {"collection_name": collection_name}) - record_count = await session.execute(count_sql) - - table_data = table_info.fetchone() - if not table_data: - return None - - return { - "table_info": { - "schemaname": table_data[0], - "tablename": table_data[1], - "tableowner": table_data[2], - "tablespace": table_data[3], - "hasindexes": table_data[4], - }, - "record_count": record_count.scalar_one(), - } - - async def delete_collection(self, collection_name: str): - async with self.db_client() as session: - async with session.begin(): - self.logger.info(f"Deleting collection: {collection_name}") - - delete_sql = sql_text(f'DROP TABLE IF EXISTS {collection_name}') - await session.execute(delete_sql) - await session.commit() - - return True - - async def create_collection(self, collection_name: str, - embedding_size: int, - do_reset: bool = False): - - if do_reset: - _ = await self.delete_collection(collection_name=collection_name) - - is_collection_existed = await self.is_collection_existed(collection_name=collection_name) - if not is_collection_existed: - self.logger.info(f"Creating collection: {collection_name}") - async with self.db_client() as session: - async with session.begin(): - create_sql = sql_text( - f'CREATE TABLE {collection_name} (' - f'{PgVectorTableSchemeEnums.ID.value} bigserial PRIMARY KEY,' - f'{PgVectorTableSchemeEnums.TEXT.value} text, ' - f'{PgVectorTableSchemeEnums.VECTOR.value} vector({embedding_size}), ' - f'{PgVectorTableSchemeEnums.METADATA.value} jsonb DEFAULT \'{{}}\', ' - f'{PgVectorTableSchemeEnums.CHUNK_ID.value} integer, ' - f'FOREIGN KEY ({PgVectorTableSchemeEnums.CHUNK_ID.value}) REFERENCES chunks(chunk_id)' - ')' - ) - await session.execute(create_sql) - await session.commit() - - return True - - return False - - async def is_index_existed(self, collection_name: str) -> bool: - index_name = self.default_index_name(collection_name) - async with self.db_client() as session: - async with session.begin(): - check_sql = sql_text(f""" - SELECT 1 - FROM pg_indexes - WHERE tablename = :collection_name - AND indexname = :index_name - """) - results = await session.execute(check_sql, {"index_name": index_name, "collection_name": collection_name}) - - return bool(results.scalar_one_or_none()) - - async def create_vector_index(self, collection_name: str, - index_type: str = PgVectorIndexTypeEnums.HNSW.value): - is_index_existed = await self.is_index_existed(collection_name=collection_name) - if is_index_existed: - return False - - async with self.db_client() as session: - async with session.begin(): - count_sql = sql_text(f'SELECT COUNT(*) FROM {collection_name}') - result = await session.execute(count_sql) - records_count = result.scalar_one() - - if records_count < self.index_threshold: - return False - - self.logger.info(f"START: Creating vector index for collection: {collection_name}") - - index_name = self.default_index_name(collection_name) - create_idx_sql = sql_text( - f'CREATE INDEX {index_name} ON {collection_name} ' - f'USING {index_type} ({PgVectorTableSchemeEnums.VECTOR.value} {self.distance_method})' - ) - - await session.execute(create_idx_sql) - - self.logger.info(f"END: Created vector index for collection: {collection_name}") - - async def reset_vector_index(self, collection_name: str, - index_type: str = PgVectorIndexTypeEnums.HNSW.value) -> bool: - - index_name = self.default_index_name(collection_name) - async with self.db_client() as session: - async with session.begin(): - drop_sql = sql_text(f'DROP INDEX IF EXISTS {index_name}') - await session.execute(drop_sql) - - return await self.create_vector_index(collection_name=collection_name, index_type=index_type) - - - async def insert_one(self, collection_name: str, text: str, vector: list, - metadata: dict = None, - record_id: str = None): - - is_collection_existed = await self.is_collection_existed(collection_name=collection_name) - if not is_collection_existed: - self.logger.error(f"Can not insert new record to non-existed collection: {collection_name}") - return False - - if not record_id: - self.logger.error(f"Can not insert new record without chunk_id: {collection_name}") - return False - - async with self.db_client() as session: - async with session.begin(): - insert_sql = sql_text(f'INSERT INTO {collection_name} ' - f'({PgVectorTableSchemeEnums.TEXT.value}, {PgVectorTableSchemeEnums.VECTOR.value}, {PgVectorTableSchemeEnums.METADATA.value}, {PgVectorTableSchemeEnums.CHUNK_ID.value}) ' - 'VALUES (:text, :vector, :metadata, :chunk_id)' - ) - - metadata_json = json.dumps(metadata, ensure_ascii=False) if metadata is not None else "{}" - await session.execute(insert_sql, { - 'text': text, - 'vector': "[" + ",".join([ str(v) for v in vector ]) + "]", - 'metadata': metadata_json, - 'chunk_id': record_id - }) - await session.commit() - - await self.create_vector_index(collection_name=collection_name) - - return True - - - async def insert_many(self, collection_name: str, texts: list, - vectors: list, metadata: list = None, - record_ids: list = None, batch_size: int = 50): - - is_collection_existed = await self.is_collection_existed(collection_name=collection_name) - if not is_collection_existed: - self.logger.error(f"Can not insert new records to non-existed collection: {collection_name}") - return False - - if len(vectors) != len(record_ids): - self.logger.error(f"Invalid data items for collection: {collection_name}") - return False - - if not metadata or len(metadata) == 0: - metadata = [None] * len(texts) - - async with self.db_client() as session: - async with session.begin(): - for i in range(0, len(texts), batch_size): - batch_texts = texts[i:i+batch_size] - batch_vectors = vectors[i:i + batch_size] - batch_metadata = metadata[i:i + batch_size] - batch_record_ids = record_ids[i:i + batch_size] - - values = [] - - for _text, _vector, _metadata, _record_id in zip(batch_texts, batch_vectors, batch_metadata, batch_record_ids): - - metadata_json = json.dumps(_metadata, ensure_ascii=False) if _metadata is not None else "{}" - values.append({ - 'text': _text, - 'vector': "[" + ",".join([ str(v) for v in _vector ]) + "]", - 'metadata': metadata_json, - 'chunk_id': _record_id - }) - - batch_insert_sql = sql_text(f'INSERT INTO {collection_name} ' - f'({PgVectorTableSchemeEnums.TEXT.value}, ' - f'{PgVectorTableSchemeEnums.VECTOR.value}, ' - f'{PgVectorTableSchemeEnums.METADATA.value}, ' - f'{PgVectorTableSchemeEnums.CHUNK_ID.value}) ' - f'VALUES (:text, :vector, :metadata, :chunk_id)') - - await session.execute(batch_insert_sql, values) - - await self.create_vector_index(collection_name=collection_name) - - return True - - async def search_by_vector(self, collection_name: str, vector: list, limit: int): - - is_collection_existed = await self.is_collection_existed(collection_name=collection_name) - if not is_collection_existed: - self.logger.error(f"Can not search for records in a non-existed collection: {collection_name}") - return False - - vector = "[" + ",".join([ str(v) for v in vector ]) + "]" - async with self.db_client() as session: - async with session.begin(): - search_sql = sql_text(f'SELECT {PgVectorTableSchemeEnums.TEXT.value} as text, 1 - ({PgVectorTableSchemeEnums.VECTOR.value} <=> :vector) as score' - f' FROM {collection_name}' - ' ORDER BY score DESC ' - f'LIMIT {limit}' - ) - - result = await session.execute(search_sql, {"vector": vector}) - - records = result.fetchall() - - return [ - RetrievedDocument( - text=record.text, - score=record.score - ) - for record in records - ] diff --git a/src/stores/vectordb/providers/QdrantDBProvider.py b/src/stores/vectordb/providers/QdrantDBProvider.py deleted file mode 100644 index 571464818..000000000 --- a/src/stores/vectordb/providers/QdrantDBProvider.py +++ /dev/null @@ -1,152 +0,0 @@ -from qdrant_client import models, QdrantClient -from ..VectorDBInterface import VectorDBInterface -from ..VectorDBEnums import DistanceMethodEnums -import logging -from typing import List -from models.db_schemes import RetrievedDocument - -class QdrantDBProvider(VectorDBInterface): - - def __init__(self, db_client: str, default_vector_size: int = 786, - distance_method: str = None, index_threshold: int=100): - - self.client = None - self.db_client = db_client - self.distance_method = None - self.default_vector_size = default_vector_size - - if distance_method == DistanceMethodEnums.COSINE.value: - self.distance_method = models.Distance.COSINE - elif distance_method == DistanceMethodEnums.DOT.value: - self.distance_method = models.Distance.DOT - - self.logger = logging.getLogger('uvicorn') - - async def connect(self): - self.client = QdrantClient(path=self.db_client) - - async def disconnect(self): - self.client = None - - async def is_collection_existed(self, collection_name: str) -> bool: - return self.client.collection_exists(collection_name=collection_name) - - async def list_all_collections(self) -> List: - return self.client.get_collections() - - def get_collection_info(self, collection_name: str) -> dict: - return self.client.get_collection(collection_name=collection_name) - - async def delete_collection(self, collection_name: str): - if self.is_collection_existed(collection_name): - self.logger.info(f"Deleting collection: {collection_name}") - return self.client.delete_collection(collection_name=collection_name) - - async def create_collection(self, collection_name: str, - embedding_size: int, - do_reset: bool = False): - if do_reset: - _ = self.delete_collection(collection_name=collection_name) - - if not self.is_collection_existed(collection_name): - self.logger.info(f"Creating new Qdrant collection: {collection_name}") - - _ = self.client.create_collection( - collection_name=collection_name, - vectors_config=models.VectorParams( - size=embedding_size, - distance=self.distance_method - ) - ) - - return True - - return False - - async def insert_one(self, collection_name: str, text: str, vector: list, - metadata: dict = None, - record_id: str = None): - - if not self.is_collection_existed(collection_name): - self.logger.error(f"Can not insert new record to non-existed collection: {collection_name}") - return False - - try: - _ = self.client.upload_records( - collection_name=collection_name, - records=[ - models.Record( - id=[record_id], - vector=vector, - payload={ - "text": text, "metadata": metadata - } - ) - ] - ) - except Exception as e: - self.logger.error(f"Error while inserting batch: {e}") - return False - - return True - - async def insert_many(self, collection_name: str, texts: list, - vectors: list, metadata: list = None, - record_ids: list = None, batch_size: int = 50): - - if metadata is None: - metadata = [None] * len(texts) - - if record_ids is None: - record_ids = list(range(0, len(texts))) - - for i in range(0, len(texts), batch_size): - batch_end = i + batch_size - - batch_texts = texts[i:batch_end] - batch_vectors = vectors[i:batch_end] - batch_metadata = metadata[i:batch_end] - batch_record_ids = record_ids[i:batch_end] - - batch_records = [ - models.Record( - id=batch_record_ids[x], - vector=batch_vectors[x], - payload={ - "text": batch_texts[x], "metadata": batch_metadata[x] - } - ) - - for x in range(len(batch_texts)) - ] - - try: - _ = self.client.upload_records( - collection_name=collection_name, - records=batch_records, - ) - except Exception as e: - self.logger.error(f"Error while inserting batch: {e}") - return False - - return True - - async def search_by_vector(self, collection_name: str, vector: list, limit: int = 5): - - results = self.client.search( - collection_name=collection_name, - query_vector=vector, - limit=limit - ) - - if not results or len(results) == 0: - return None - - return [ - RetrievedDocument(**{ - "score": result.score, - "text": result.payload["text"], - }) - for result in results - ] - diff --git a/src/stores/vectordb/providers/__init__.py b/src/stores/vectordb/providers/__init__.py deleted file mode 100644 index 75bfb8e1a..000000000 --- a/src/stores/vectordb/providers/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .QdrantDBProvider import QdrantDBProvider -from .PGVectorProvider import PGVectorProvider diff --git a/src/tasks/__init__.py b/src/tasks/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/tasks/data_indexing.py b/src/tasks/data_indexing.py deleted file mode 100644 index a93b5f360..000000000 --- a/src/tasks/data_indexing.py +++ /dev/null @@ -1,144 +0,0 @@ -from celery_app import celery_app, get_setup_utils -from helpers.config import get_settings -import asyncio -from fastapi.responses import JSONResponse -from models.ProjectModel import ProjectModel -from models.ChunkModel import ChunkModel -from controllers import NLPController -from models import ResponseSignal -from tqdm.auto import tqdm - -import logging -logger = logging.getLogger(__name__) - -@celery_app.task( - bind=True, name="tasks.data_indexing.index_data_content", - autoretry_for=(Exception,), - retry_kwargs={'max_retries': 3, 'countdown': 60} - ) -def index_data_content(self, project_id: int, do_reset: int): - - logger.warning("index_data_content started") - return asyncio.run( - _index_data_content(self, project_id, do_reset) - ) - -async def _index_data_content(task_instance, project_id: int, do_reset: int): - - db_engine, vectordb_client = None, None - - try: - - (db_engine, db_client, llm_provider_factory, - vectordb_provider_factory, - generation_client, embedding_client, - vectordb_client, template_parser) = await get_setup_utils() - - logger.warning("Setup utils were loaded!") - - project_model = await ProjectModel.create_instance( - db_client=db_client - ) - - chunk_model = await ChunkModel.create_instance( - db_client=db_client - ) - - project = await project_model.get_project_or_create_one( - project_id=project_id - ) - - if not project: - - task_instance.update_state( - state="FAILURE", - meta={ - "signal": ResponseSignal.PROJECT_NOT_FOUND_ERROR.value - } - ) - - raise Exception(f"No project found for project_id: {project_id}") - - nlp_controller = NLPController( - vectordb_client=vectordb_client, - generation_client=generation_client, - embedding_client=embedding_client, - template_parser=template_parser, - ) - - has_records = True - page_no = 1 - inserted_items_count = 0 - idx = 0 - - # create collection if not exists - collection_name = nlp_controller.create_collection_name(project_id=project.project_id) - - _ = await vectordb_client.create_collection( - collection_name=collection_name, - embedding_size=embedding_client.embedding_size, - do_reset=do_reset, - ) - - # setup batching - total_chunks_count = await chunk_model.get_total_chunks_count(project_id=project.project_id) - pbar = tqdm(total=total_chunks_count, desc="Vector Indexing", position=0) - - while has_records: - page_chunks = await chunk_model.get_poject_chunks(project_id=project.project_id, page_no=page_no) - if len(page_chunks): - page_no += 1 - - if not page_chunks or len(page_chunks) == 0: - has_records = False - break - - chunks_ids = [ c.chunk_id for c in page_chunks ] - idx += len(page_chunks) - - is_inserted = await nlp_controller.index_into_vector_db( - project=project, - chunks=page_chunks, - chunks_ids=chunks_ids - ) - - if not is_inserted: - - - task_instance.update_state( - state="FAILURE", - meta={ - "signal": ResponseSignal.INSERT_INTO_VECTORDB_ERROR.value - } - ) - - raise Exception(f"can not insert into vectorDB | project_id: {project_id}") - - pbar.update(len(page_chunks)) - inserted_items_count += len(page_chunks) - - - task_instance.update_state( - state="SUCCESS", - meta={ - "signal": ResponseSignal.INSERT_INTO_VECTORDB_SUCCESS.value, - } - ) - - return { - "signal": ResponseSignal.INSERT_INTO_VECTORDB_SUCCESS.value, - "inserted_items_count": inserted_items_count - } - - except Exception as e: - logger.error(f"Task failed: {str(e)}") - raise - finally: - try: - if db_engine: - await db_engine.dispose() - - if vectordb_client: - await vectordb_client.disconnect() - except Exception as e: - logger.error(f"Task failed while cleaning: {str(e)}") \ No newline at end of file diff --git a/src/tasks/file_processing.py b/src/tasks/file_processing.py deleted file mode 100644 index 361ce98c1..000000000 --- a/src/tasks/file_processing.py +++ /dev/null @@ -1,262 +0,0 @@ -from celery_app import celery_app, get_setup_utils -from helpers.config import get_settings -import asyncio -from models.ProjectModel import ProjectModel -from models.ChunkModel import ChunkModel -from models.AssetModel import AssetModel -from models.db_schemes import DataChunk -from models import ResponseSignal -from models.enums.AssetTypeEnum import AssetTypeEnum -from controllers import ProcessController -from controllers import NLPController -from utils.idempotency_manager import IdempotencyManager - -import logging -logger = logging.getLogger(__name__) - -@celery_app.task( - bind=True, name="tasks.file_processing.process_project_files", - autoretry_for=(Exception,), - retry_kwargs={'max_retries': 3, 'countdown': 60} - ) -def process_project_files(self, project_id: int, - file_id: int, chunk_size: int, - overlap_size: int, do_reset: int): - - return asyncio.run( - _process_project_files(self, project_id, file_id, chunk_size, - overlap_size, do_reset) - ) - - -async def _process_project_files(task_instance, project_id: int, - file_id: int, chunk_size: int, - overlap_size: int, do_reset: int): - - - db_engine, vectordb_client = None, None - - try: - - (db_engine, db_client, llm_provider_factory, - vectordb_provider_factory, - generation_client, embedding_client, - vectordb_client, template_parser) = await get_setup_utils() - - # Create idempotency manager - idempotency_manager = IdempotencyManager(db_client, db_engine) - - # Define task arguments for idempotency check - task_args = { - "project_id": project_id, - "file_id": file_id, - "chunk_size": chunk_size, - "overlap_size": overlap_size, - "do_reset": do_reset - } - - task_name = "tasks.file_processing.process_project_files" - - settings = get_settings() - - # Check if task should execute (600 seconds = 10 minutes timeout) - should_execute, existing_task = await idempotency_manager.should_execute_task( - task_name=task_name, - task_args=task_args, - celery_task_id=task_instance.request.id, - task_time_limit=settings.CELERY_TASK_TIME_LIMIT - ) - - if not should_execute: - logger.warning(f"Can not handle th task | status: {existing_task.status}") - return existing_task.result - - task_record = None - if existing_task: - # Update existing task with new celery task ID - await idempotency_manager.update_task_status( - execution_id=existing_task.execution_id, - status='PENDING' - ) - task_record = existing_task - else: - # Create new task record - task_record = await idempotency_manager.create_task_record( - task_name=task_name, - task_args=task_args, - celery_task_id=task_instance.request.id - ) - - # Update status to STARTED - await idempotency_manager.update_task_status( - execution_id=task_record.execution_id, - status='STARTED' - ) - - - project_model = await ProjectModel.create_instance( - db_client=db_client - ) - - project = await project_model.get_project_or_create_one( - project_id=project_id - ) - - nlp_controller = NLPController( - vectordb_client=vectordb_client, - generation_client=generation_client, - embedding_client=embedding_client, - template_parser=template_parser, - ) - - asset_model = await AssetModel.create_instance( - db_client=db_client - ) - - project_files_ids = {} - if file_id: - asset_record = await asset_model.get_asset_record( - asset_project_id=project.project_id, - asset_name=file_id - ) - - if asset_record is None: - task_instance.update_state( - state="FAILURE", - meta={ - "signal": ResponseSignal.FILE_ID_ERROR.value, - } - ) - - # Update task status to FAILURE - await idempotency_manager.update_task_status( - execution_id=task_record.execution_id, - status='FAILURE', - result={"signal": ResponseSignal.FILE_ID_ERROR.value} - ) - - raise Exception(f"No assets for file: {file_id}") - - project_files_ids = { - asset_record.asset_id: asset_record.asset_name - } - - else: - - - project_files = await asset_model.get_all_project_assets( - asset_project_id=project.project_id, - asset_type=AssetTypeEnum.FILE.value, - ) - - project_files_ids = { - record.asset_id: record.asset_name - for record in project_files - } - - if len(project_files_ids) == 0: - - task_instance.update_state( - state="FAILURE", - meta={ - "signal": ResponseSignal.NO_FILES_ERROR.value, - } - ) - - # Update task status to FAILURE - await idempotency_manager.update_task_status( - execution_id=task_record.execution_id, - status='FAILURE', - result={"signal": ResponseSignal.NO_FILES_ERROR.value,} - ) - - raise Exception(f"No files found for project_id: {project.project_id}") - - process_controller = ProcessController(project_id=project_id) - - no_records = 0 - no_files = 0 - - chunk_model = await ChunkModel.create_instance( - db_client=db_client - ) - - if do_reset == 1: - # delete associated vectors collection - collection_name = nlp_controller.create_collection_name(project_id=project.project_id) - _ = await vectordb_client.delete_collection(collection_name=collection_name) - - # delete associated chunks - _ = await chunk_model.delete_chunks_by_project_id( - project_id=project.project_id - ) - - for asset_id, file_id in project_files_ids.items(): - - file_content = process_controller.get_file_content(file_id=file_id) - - if file_content is None: - logger.error(f"Error while processing file: {file_id}") - continue - - file_chunks = process_controller.process_file_content( - file_content=file_content, - file_id=file_id, - chunk_size=chunk_size, - overlap_size=overlap_size - ) - - if file_chunks is None or len(file_chunks) == 0: - - logger.error(f"No chunks for file_id: {file_id}") - pass - - file_chunks_records = [ - DataChunk( - chunk_text=chunk.page_content, - chunk_metadata=chunk.metadata, - chunk_order=i+1, - chunk_project_id=project.project_id, - chunk_asset_id=asset_id - ) - for i, chunk in enumerate(file_chunks) - ] - - no_records += await chunk_model.insert_many_chunks(chunks=file_chunks_records) - no_files += 1 - - task_instance.update_state( - state="SUCCESS", - meta={ - "signal": ResponseSignal.PROCESSING_SUCCESS.value, - } - ) - - await idempotency_manager.update_task_status( - execution_id=task_record.execution_id, - status='SUCCESS', - result={"signal": ResponseSignal.PROCESSING_SUCCESS.value} - ) - - logger.warning(f"inserted_chunks: {no_records}") - - return { - "signal": ResponseSignal.PROCESSING_SUCCESS.value, - "inserted_chunks": no_records, - "processed_files": no_files, - "project_id": project_id, - "do_reset": do_reset - } - - except Exception as e: - logger.error(f"Task failed: {str(e)}") - raise - finally: - try: - if db_engine: - await db_engine.dispose() - - if vectordb_client: - await vectordb_client.disconnect() - except Exception as e: - logger.error(f"Task failed while cleaning: {str(e)}") \ No newline at end of file diff --git a/src/tasks/maintenance.py b/src/tasks/maintenance.py deleted file mode 100644 index 6b2e94ea9..000000000 --- a/src/tasks/maintenance.py +++ /dev/null @@ -1,50 +0,0 @@ -from celery_app import celery_app, get_setup_utils -from helpers.config import get_settings -import asyncio -from utils.idempotency_manager import IdempotencyManager - -import logging -logger = logging.getLogger(__name__) - -@celery_app.task( - bind=True, name="tasks.maintenance.clean_celery_executions_table", - autoretry_for=(Exception,), - retry_kwargs={'max_retries': 3, 'countdown': 60} - ) -def clean_celery_executions_table(self): - - return asyncio.run( - _clean_celery_executions_table(self) - ) - -async def _clean_celery_executions_table(task_instance): - - db_engine, vectordb_client = None, None - - try: - - (db_engine, db_client, llm_provider_factory, - vectordb_provider_factory, - generation_client, embedding_client, - vectordb_client, template_parser) = await get_setup_utils() - - # Create idempotency manager - idempotency_manager = IdempotencyManager(db_client, db_engine) - - logger.warning(f"cleaning !!!") - _ = await idempotency_manager.cleanup_old_tasks(5) - - return True - - except Exception as e: - logger.error(f"Task failed: {str(e)}") - raise - finally: - try: - if db_engine: - await db_engine.dispose() - - if vectordb_client: - await vectordb_client.disconnect() - except Exception as e: - logger.error(f"Task failed while cleaning: {str(e)}") \ No newline at end of file diff --git a/src/tasks/process_workflow.py b/src/tasks/process_workflow.py deleted file mode 100644 index aa77b61fc..000000000 --- a/src/tasks/process_workflow.py +++ /dev/null @@ -1,54 +0,0 @@ -from celery import chain -from celery_app import celery_app, get_setup_utils -from helpers.config import get_settings -import asyncio -from tasks.file_processing import process_project_files -from tasks.data_indexing import _index_data_content - -import logging -logger = logging.getLogger(__name__) - -@celery_app.task( - bind=True, name="tasks.process_workflow.push_after_process_task", - autoretry_for=(Exception,), - retry_kwargs={'max_retries': 3, 'countdown': 60} - ) -def push_after_process_task(self, prev_task_result): - - project_id = prev_task_result.get("project_id") - do_reset = prev_task_result.get("do_reset") - - task_results = asyncio.run( - _index_data_content(self, project_id, do_reset) - ) - - return { - "project_id": project_id, - "do_reset": do_reset, - "task_results": task_results - } - - -@celery_app.task( - bind=True, name="tasks.process_workflow.process_and_push_workflow", - autoretry_for=(Exception,), - retry_kwargs={'max_retries': 3, 'countdown': 60} - ) -def process_and_push_workflow( self, project_id: int, - file_id: int, chunk_size: int, - overlap_size: int, do_reset: int): - - workflow = chain( - process_project_files.s(project_id, file_id, chunk_size, overlap_size, do_reset), - push_after_process_task.s() - ) - - result = workflow.apply_async() - - return { - "signal": "WORKFLOW_STARTED", - "workflow_id": result.id, - "tasks": ["tasks.file_processing.process_project_files", - "tasks.data_indexing.index_data_content"] - } - diff --git a/src/utils/__init__.py b/src/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/utils/idempotency_manager.py b/src/utils/idempotency_manager.py deleted file mode 100644 index 127a14dea..000000000 --- a/src/utils/idempotency_manager.py +++ /dev/null @@ -1,124 +0,0 @@ -import hashlib -import json -from datetime import datetime, timedelta, timezone -from sqlalchemy import select, delete -from models.db_schemes.minirag.schemes.celery_task_execution import CeleryTaskExecution - -class IdempotencyManager: - - def __init__(self, db_client, db_engine): - self.db_client = db_client - self.db_engine = db_engine - - def create_args_hash(self, task_name: str, task_args: dict): - combined_data = { - **task_args, - "task_name": task_name - } - json_string = json.dumps(combined_data, sort_keys=True, default=str) - return hashlib.sha256(json_string.encode()).hexdigest() - - async def create_task_record(self, task_name: str, task_args: dict, celery_task_id: str = None) -> CeleryTaskExecution: - """Create new task execution record.""" - args_hash = self.create_args_hash(task_name, task_args) - - task_record = CeleryTaskExecution( - task_name=task_name, - task_args_hash=args_hash, - task_args=task_args, - celery_task_id=celery_task_id, - status='PENDING', - started_at=datetime.utcnow() - ) - - session = self.db_client() - try: - session.add(task_record) - await session.commit() - await session.refresh(task_record) - return task_record - finally: - await session.close() - - async def update_task_status(self, execution_id: int, status: str, result: dict = None): - """Update task status and result.""" - session = self.db_client() - try: - task_record = await session.get(CeleryTaskExecution, execution_id) - if task_record: - task_record.status = status - if result: - task_record.result = result - if status in ['SUCCESS', 'FAILURE']: - task_record.completed_at = datetime.utcnow() - await session.commit() - finally: - await session.close() - - async def get_existing_task(self, task_name: str, - task_args: dict, celery_task_id: str) -> CeleryTaskExecution: - """Check if task with same name and args already exists.""" - args_hash = self.create_args_hash(task_name, task_args) - - session = self.db_client() - try: - stmt = select(CeleryTaskExecution).where( - CeleryTaskExecution.celery_task_id == celery_task_id, - CeleryTaskExecution.task_name == task_name, - CeleryTaskExecution.task_args_hash == args_hash - ) - result = await session.execute(stmt) - return result.scalar_one_or_none() - finally: - await session.close() - - async def should_execute_task(self, task_name: str, task_args: dict, - celery_task_id: str, - task_time_limit: int = 600) -> tuple[bool, CeleryTaskExecution]: - """ - Check if task should be executed or return existing result. - Args: - task_time_limit: Time limit in seconds after which a stuck task can be re-executed - Returns (should_execute, existing_task_or_none) - """ - existing_task = await self.get_existing_task(task_name, task_args, celery_task_id) - - if not existing_task: - return True, None - - # Don't execute if task is already completed successfully - if existing_task.status == 'SUCCESS': - return False, existing_task - - # Check if task is stuck (running longer than time limit + 60 seconds) - if existing_task.status in ['PENDING', 'STARTED', 'RETRY']: - if existing_task.started_at: - time_elapsed = (datetime.utcnow() - existing_task.started_at).total_seconds() - time_gap = 60 # 60 seconds grace period - if time_elapsed > (task_time_limit + time_gap): - return True, existing_task # Task is stuck, allow re-execution - return False, existing_task # Task is still running within time limit - - # Re-execute if previous task failed - return True, existing_task - - async def cleanup_old_tasks(self, time_retention: int = 86400) -> int: - """ - Delete old task records older than time_retention seconds. - Args: - time_retention: Time in seconds to retain tasks (default: 86400 = 24 hours) - Returns: - Number of deleted records - """ - cutoff_time = datetime.now(timezone.utc) - timedelta(seconds=time_retention) - - session = self.db_client() - try: - stmt = delete(CeleryTaskExecution).where( - CeleryTaskExecution.created_at < cutoff_time - ) - result = await session.execute(stmt) - await session.commit() - return result.rowcount - finally: - await session.close() \ No newline at end of file diff --git a/src/utils/metrics.py b/src/utils/metrics.py deleted file mode 100644 index 1b189f3d2..000000000 --- a/src/utils/metrics.py +++ /dev/null @@ -1,36 +0,0 @@ -from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST -from fastapi import FastAPI, Request, Response -from starlette.middleware.base import BaseHTTPMiddleware -import time - -# Define metrics -REQUEST_COUNT = Counter('http_requests_total', 'Total HTTP Requests', ['method', 'endpoint', 'status']) -REQUEST_LATENCY = Histogram('http_request_duration_seconds', 'HTTP Request Latency', ['method', 'endpoint']) - -class PrometheusMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - - start_time = time.time() - - # Process the request - response = await call_next(request) - - # Record metrics after request is processed - duration = time.time() - start_time - endpoint = request.url.path - - REQUEST_LATENCY.labels(method=request.method, endpoint=endpoint).observe(duration) - REQUEST_COUNT.labels(method=request.method, endpoint=endpoint, status=response.status_code).inc() - - return response - -def setup_metrics(app: FastAPI): - """ - Setup Prometheus metrics middleware and endpoint - """ - # Add Prometheus middleware - app.add_middleware(PrometheusMiddleware) - - @app.get("/TrhBVe_m5gg2002_E5VVqS", include_in_schema=False) - def metrics(): - return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST) From ecbdbd45e0d561944a90c240ad2f8777b0194aa7 Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Sun, 15 Feb 2026 05:13:31 +0200 Subject: [PATCH 2/8] update .gitignore --- .gitignore | 217 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..2975f09d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,217 @@ +# Created by venv; see https://docs.python.org/3/library/venv.html +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml \ No newline at end of file From 227f439bd0f5b9992cdf491d5b57e1635f5d424e Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Sun, 15 Feb 2026 05:19:06 +0200 Subject: [PATCH 3/8] update readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..24f4c43b3 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +#mini-rag + +this is a minimal implementation of the RAG model for quistion answering. + +## Requirments + +- Python 3.8 or later From 9c490e47908df3e76c2cd8773449480eddcae58b Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Sun, 15 Feb 2026 05:46:10 +0200 Subject: [PATCH 4/8] update requiremments --- requirment.txt | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 requirment.txt diff --git a/requirment.txt b/requirment.txt new file mode 100644 index 000000000..0389cc365 --- /dev/null +++ b/requirment.txt @@ -0,0 +1,33 @@ +fastapi==0.110.2 +uvicorn[standard]==0.29.0 + +python-multipart==0.0.9 +python-dotenv==1.0.1 +pydantic-settings==2.2.1 +aiofiles==23.2.1 +langchain==0.1.20 +PyMuPDF==1.24.3 +motor==3.4.0 +pydantic-mongo==2.3.0 +openai==1.75.0 +cohere==5.5.8 +qdrant-client==1.10.1 +SQLAlchemy==2.0.36 +asyncpg==0.30.0 +alembic==1.14.0 +psycopg2-binary==2.9.10 +pgvector==0.4.0 +nltk==3.9.1 + +# Monitoring and metrics +prometheus-client==0.21.1 +starlette-exporter==0.23.0 +fastapi-health==0.4.0 + +# Task Queue and Background Processing +celery==5.5.3 +redis==6.2.0 +kombu==5.5.4 +billiard==4.2.1 +vine==5.1.0 +flower==2.0.1 \ No newline at end of file From 5697de4bcb2c6d4572c5dffe7f09960ec22e95a5 Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Sun, 15 Feb 2026 05:48:25 +0200 Subject: [PATCH 5/8] create gitkeep --- assets/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 assets/.gitkeep diff --git a/assets/.gitkeep b/assets/.gitkeep new file mode 100644 index 000000000..e69de29bb From 8b68f694f73565e382f5fe2d44384fcb5e958ec8 Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Sun, 15 Feb 2026 05:59:52 +0200 Subject: [PATCH 6/8] create env example file --- .env.example | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..cbd3aa708 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +APP_NAME="mini_RAG" +APP_VERSION="0.1" From 1851a7c036bf268f6e0940970b0fb09bb6818f27 Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Sun, 15 Feb 2026 07:54:43 +0200 Subject: [PATCH 7/8] create main file with Fastapi code --- main.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 main.py diff --git a/main.py b/main.py new file mode 100644 index 000000000..135337233 --- /dev/null +++ b/main.py @@ -0,0 +1,7 @@ +from fastapi import FastAPI + +app = FastAPI() + +@app.get("/") +def welcom (): + return {"Welcome to FastAPI!"} \ No newline at end of file From 09915044649afd2b35c9f9140b636ed054caac1f Mon Sep 17 00:00:00 2001 From: Abdelrahman Mahmoud Date: Mon, 16 Feb 2026 02:43:35 +0200 Subject: [PATCH 8/8] add postman collection --- assets/mini-rag-app.postman_collection.json | 57 +++++++++++++++++++++ main.py | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 assets/mini-rag-app.postman_collection.json diff --git a/assets/mini-rag-app.postman_collection.json b/assets/mini-rag-app.postman_collection.json new file mode 100644 index 000000000..57e1b9525 --- /dev/null +++ b/assets/mini-rag-app.postman_collection.json @@ -0,0 +1,57 @@ +{ + "info": { + "_postman_id": "10b9e88a-c864-46a6-835a-8545b1db4899", + "name": "mini-rag-app", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "52414106", + "_collection_link": "https://go.postman.co/collection/52414106-10b9e88a-c864-46a6-835a-8545b1db4899?source=collection_link" + }, + "item": [ + { + "name": "welcom-requist", + "request": { + "method": "GET", + "header": [] + }, + "response": [] + }, + { + "name": "New Request", + "request": { + "method": "GET", + "header": [] + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "packages": {}, + "requests": {}, + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "packages": {}, + "requests": {}, + "exec": [ + "" + ] + } + } + ], + "variable": [ + { + "key": "api", + "value": "" + } + ] +} \ No newline at end of file diff --git a/main.py b/main.py index 135337233..1c2f609ae 100644 --- a/main.py +++ b/main.py @@ -4,4 +4,4 @@ @app.get("/") def welcom (): - return {"Welcome to FastAPI!"} \ No newline at end of file + return {"massege":"Welcome to FastAPI!"} \ No newline at end of file