From cf5e68dac9c1963f035e5a21bfda08cad1b55d6c Mon Sep 17 00:00:00 2001 From: Ruby Bui Date: Sat, 10 May 2025 23:10:14 -0500 Subject: [PATCH 1/3] add rate limiter using memcache --- Dockerfile | 9 +++++++++ docker-compose.yml | 22 ++++++++++++++++++++++ mind_matter_api/app.py | 11 +++++++++-- mind_matter_api/middleware/rate_limit.py | 22 ++++++++++++++++++++++ requirements/dev.txt | 2 +- requirements/prod.txt | 8 +++++--- 6 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 mind_matter_api/middleware/rate_limit.py diff --git a/Dockerfile b/Dockerfile index 5134810..8720065 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,15 @@ COPY mind_matter_api mind_matter_api COPY .env.example .env +# ================================= DEVELOPMENT ================================ +FROM builder AS development +RUN pip install --no-cache -r requirements/dev.txt +EXPOSE 2992 +EXPOSE 5000 + +CMD [ "flask", "run", "--host=0.0.0.0"] + + # ================================= PRODUCTION ================================= FROM python:${INSTALL_PYTHON_VERSION}-slim-bullseye as production diff --git a/docker-compose.yml b/docker-compose.yml index 14f28cb..9c72178 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,12 @@ services: ports: - "5000:5000" - "2992:2992" + environment: + CACHE_MEMCACHED_SERVERS: memcached:11211 + FLASK_ENV: development + FLASK_DEBUG: 1 + depends_on: + - memcached <<: *default_volumes mind-matter-flask-prod: @@ -32,6 +38,9 @@ services: FLASK_DEBUG: 0 LOG_LEVEL: info GUNICORN_WORKERS: 4 + CACHE_MEMCACHED_SERVERS: memcached:11211 + depends_on: + - memcached <<: *default_volumes mind-matter-manage: @@ -48,3 +57,16 @@ services: stdin_open: true tty: true <<: *default_volumes + + memcached: + image: memcached:alpine + ports: + - "11211:11211" + restart: unless-stopped + healthcheck: + test: ["CMD", "echo", "version", "|", "nc", "localhost", "11211"] + interval: 5s + timeout: 2s + retries: 5 + + diff --git a/mind_matter_api/app.py b/mind_matter_api/app.py index ce62386..67574b6 100644 --- a/mind_matter_api/app.py +++ b/mind_matter_api/app.py @@ -13,6 +13,7 @@ from mind_matter_api.api import register_routes from mind_matter_api.schemas import UserSchema, UserBodySchema +from mind_matter_api.middleware.rate_limit import rate_limit_middleware from mind_matter_api.extensions import ( cache, csrf_protect, @@ -32,10 +33,16 @@ def create_app(config_object="mind_matter_api.settings"): :param config_object: The configuration object to use. """ app = Flask(__name__, root_path=os.path.dirname(os.path.abspath(__file__))) - CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True) # <--- THIS LINE - + CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True) # < + + # Rate limiter using flask-caching (memcached ) + logger = logging.getLogger(__name__) + if not hasattr(cache, '_cache'): # or check if cache.cache is None + cache.init_app(app) + + app.before_request(rate_limit_middleware(limit=5, window=60)) # Load configuration app.config.from_object(config_object) app.config["SQLALCHEMY_RECORD_QUERIES"] = True diff --git a/mind_matter_api/middleware/rate_limit.py b/mind_matter_api/middleware/rate_limit.py new file mode 100644 index 0000000..432238b --- /dev/null +++ b/mind_matter_api/middleware/rate_limit.py @@ -0,0 +1,22 @@ +import time +from flask import request, jsonify +from mind_matter_api.extensions import cache +def rate_limit_middleware(limit: int, window: int): + def middleware(): + ip = request.remote_addr or "global" + key = f"rate-limit:{ip}:{request.endpoint}" + now = time.time() + + record = cache.get(key) + + if record: + count, first_time = record + if now - first_time < window: + if count >= limit: + return jsonify({"error": "Rate limit exceeded"}), 429 + cache.set(key, (count + 1, first_time)) + else: + cache.set(key, (1, now)) + else: + cache.set(key, (1, now)) + return middleware diff --git a/requirements/dev.txt b/requirements/dev.txt index ecae690..c48c8f3 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -19,4 +19,4 @@ pep8-naming==0.14.1 Flask-Cors==5.0.0 -pyjwt==2.10.1 \ No newline at end of file + diff --git a/requirements/prod.txt b/requirements/prod.txt index 3217bb9..b9d4b02 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -17,7 +17,7 @@ Flask-Migrate==4.0.7 # Handles database migrations with Alembic # === Authentication and User Management === Flask-Login==0.6.3 # User session management for Flask -PyJWT==2.8.0 # JSON Web Token implementation +pyjwt==2.10.1 # === Deployment === gevent==24.11.1 # Asynchronous networking library for concurrent requests @@ -25,8 +25,6 @@ gunicorn>=19.9.0 # WSGI HTTP server for deploying Python applications supervisor==4.2.5 # Process manager for running background services -# === Caching === -Flask-Caching>=2.0.2 # Adds caching support for better performance # === Environment Variable Management === environs==11.2.1 # Simplifies parsing and managing environment variables @@ -52,4 +50,8 @@ pytest-cov==6.0.0 WebTest==3.0.2 flasgger + Flask-Mail + +# === Rate Limiting === +Flask-Caching==2.3.1 From 6f0a6e07bdbd277b80c5f00ef3c1ec2c71361c0d Mon Sep 17 00:00:00 2001 From: Ruby Bui Date: Sun, 11 May 2025 01:24:37 -0500 Subject: [PATCH 2/3] change to use simple cache ,add exception --- docker-compose.yml | 21 ++------------------- mind_matter_api/app.py | 11 ++++++----- mind_matter_api/middleware/rate_limit.py | 15 +++++++++++++-- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9c72178..a04aac8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,11 +17,10 @@ services: - "5000:5000" - "2992:2992" environment: - CACHE_MEMCACHED_SERVERS: memcached:11211 + FLASK_ENV: development FLASK_DEBUG: 1 - depends_on: - - memcached + <<: *default_volumes mind-matter-flask-prod: @@ -38,9 +37,6 @@ services: FLASK_DEBUG: 0 LOG_LEVEL: info GUNICORN_WORKERS: 4 - CACHE_MEMCACHED_SERVERS: memcached:11211 - depends_on: - - memcached <<: *default_volumes mind-matter-manage: @@ -57,16 +53,3 @@ services: stdin_open: true tty: true <<: *default_volumes - - memcached: - image: memcached:alpine - ports: - - "11211:11211" - restart: unless-stopped - healthcheck: - test: ["CMD", "echo", "version", "|", "nc", "localhost", "11211"] - interval: 5s - timeout: 2s - retries: 5 - - diff --git a/mind_matter_api/app.py b/mind_matter_api/app.py index 67574b6..88de66d 100644 --- a/mind_matter_api/app.py +++ b/mind_matter_api/app.py @@ -33,16 +33,12 @@ def create_app(config_object="mind_matter_api.settings"): :param config_object: The configuration object to use. """ app = Flask(__name__, root_path=os.path.dirname(os.path.abspath(__file__))) - CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True) # < + CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=False) # < # Rate limiter using flask-caching (memcached ) logger = logging.getLogger(__name__) - if not hasattr(cache, '_cache'): # or check if cache.cache is None - cache.init_app(app) - - app.before_request(rate_limit_middleware(limit=5, window=60)) # Load configuration app.config.from_object(config_object) app.config["SQLALCHEMY_RECORD_QUERIES"] = True @@ -56,6 +52,11 @@ def create_app(config_object="mind_matter_api.settings"): register_routes(app) configure_logger(app) + app.before_request(rate_limit_middleware( + limit=5, + window=60, + exclude_paths=["/health"] + )) # Swagger UI Swagger( diff --git a/mind_matter_api/middleware/rate_limit.py b/mind_matter_api/middleware/rate_limit.py index 432238b..98a26ba 100644 --- a/mind_matter_api/middleware/rate_limit.py +++ b/mind_matter_api/middleware/rate_limit.py @@ -1,8 +1,18 @@ import time from flask import request, jsonify from mind_matter_api.extensions import cache -def rate_limit_middleware(limit: int, window: int): + + +def rate_limit_middleware(limit: int, window: int, exclude_paths=None, exclude_methods=None): + exclude_paths = exclude_paths or [] + exclude_methods = exclude_methods or [] + def middleware(): + cache.set("foo", "bar") + print(cache.get("foo")) + if request.path in exclude_paths or request.method in exclude_methods: + return + ip = request.remote_addr or "global" key = f"rate-limit:{ip}:{request.endpoint}" now = time.time() @@ -19,4 +29,5 @@ def middleware(): cache.set(key, (1, now)) else: cache.set(key, (1, now)) - return middleware + + return middleware \ No newline at end of file From ad54364d00ae0d2db4b4f515c828fafcc3583760 Mon Sep 17 00:00:00 2001 From: Ruby Bui Date: Sun, 11 May 2025 01:36:33 -0500 Subject: [PATCH 3/3] identity based on devices --- mind_matter_api/middleware/rate_limit.py | 48 ++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/mind_matter_api/middleware/rate_limit.py b/mind_matter_api/middleware/rate_limit.py index 98a26ba..c0fe2b1 100644 --- a/mind_matter_api/middleware/rate_limit.py +++ b/mind_matter_api/middleware/rate_limit.py @@ -1,6 +1,28 @@ import time from flask import request, jsonify -from mind_matter_api.extensions import cache +from mind_matter_api.extensions import cache +from mind_matter_api.utils.auth import decode_auth_token # assume this exists + +def get_rate_limit_identity(): + # 1. Check for Device ID header (mobile apps) + device_id = request.headers.get("X-Device-ID") + if device_id: + return f"device:{device_id}" + + # 2. Check for authenticated user (e.g. JWT bearer) + auth_header = request.headers.get("Authorization") + if auth_header and " " in auth_header: + token_type, token = auth_header.split(" ", 1) + if token_type.lower() == "bearer": + try: + user_id = decode_auth_token(token) + if user_id: + return f"user:{user_id}" + except Exception: + pass + + # 3. Fallback to IP address + return f"ip:{request.remote_addr or 'unknown'}" def rate_limit_middleware(limit: int, window: int, exclude_paths=None, exclude_methods=None): @@ -8,13 +30,11 @@ def rate_limit_middleware(limit: int, window: int, exclude_paths=None, exclude_m exclude_methods = exclude_methods or [] def middleware(): - cache.set("foo", "bar") - print(cache.get("foo")) if request.path in exclude_paths or request.method in exclude_methods: - return + return - ip = request.remote_addr or "global" - key = f"rate-limit:{ip}:{request.endpoint}" + identity = get_rate_limit_identity() + key = f"rate-limit:{identity}:{request.endpoint}" now = time.time() record = cache.get(key) @@ -23,11 +43,17 @@ def middleware(): count, first_time = record if now - first_time < window: if count >= limit: - return jsonify({"error": "Rate limit exceeded"}), 429 - cache.set(key, (count + 1, first_time)) + retry_after = int(window - (now - first_time) + 1) + response = jsonify({ + "error": "Rate limit exceeded", + "retry_after_seconds": retry_after + }) + response.headers["Retry-After"] = str(retry_after) + return response, 429 + cache.set(key, (count + 1, first_time), timeout=window * 2) else: - cache.set(key, (1, now)) + cache.set(key, (1, now), timeout=window * 2) else: - cache.set(key, (1, now)) + cache.set(key, (1, now), timeout=window * 2) - return middleware \ No newline at end of file + return middleware