Migrate from uWSGI to Gunicorn

.github/workflows/pull_request_automation.yml

@@ -101,19 +101,19 @@ jobs:
         if: always()
         run: |
           docker ps -a
-          docker logs intelowl_uwsgi
+          docker logs intelowl_gunicorn
           docker logs intelowl_daphne
 
       - name: Setup coverage
         run: |
-          docker exec intelowl_uwsgi pip3 install coverage
+          docker exec intelowl_gunicorn pip3 install coverage
 
       - name: Run test
         run: |
-          docker exec intelowl_uwsgi coverage run manage.py test --keepdb tests
+          docker exec intelowl_gunicorn coverage run manage.py test --keepdb tests
       - name: Run async tests
         run: |
-          docker exec intelowl_uwsgi coverage run manage.py test --keepdb async_tests
+          docker exec intelowl_gunicorn coverage run manage.py test --keepdb async_tests
 
   frontend-tests:
     runs-on: ubuntu-latest

api_app/management/commands/elastic_templates.py

@@ -9,7 +9,7 @@
 
 
 class Command(BaseCommand):
-    # NOTE: this command is runned by uwsgi startup script
+    # NOTE: this command is run by gunicorn startup script
 
     help = "Create or update the index templates in Elasticsearch"
 

configuration/gunicorn.conf.py

@@ -0,0 +1,31 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+import multiprocessing
+
+# Gunicorn configuration file for IntelOwl
+
+bind = "0.0.0.0:8001"
+worker_class = "gthread"
+
+# Dynamic worker calculation: (2 x num_cores) + 1
+workers = multiprocessing.cpu_count() * 2 + 1
+threads = 2
+
+timeout = 600
+
+max_requests = 1000
+max_requests_jitter = 50
+keepalive = 5
+preload_app = True
+
+# Standard Docker logging to stdout/stderr
+accesslog = "-"
+errorlog = "-"
+loglevel = "info"
+
+proc_name = "intel_owl_gunicorn"
+
+# Handle headers from Nginx correctly
+forwarded_allow_ips = "*"
+proxy_allow_ips = "*"

configuration/nginx/django_server.conf

@@ -10,14 +10,14 @@ server {
         log_not_found off; 
     }
 
-    # All requests to the Django/UWSGI server.
+    # All requests to the Django/Gunicorn server.
     location / {
         proxy_set_header        X-Forwarded-Proto https;
         proxy_set_header        X-Url-Scheme $scheme;
         proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header        Host $http_host;
         proxy_redirect          off;
-        proxy_pass              http://uwsgi:8001;
+        proxy_pass              http://gunicorn:8001;
         client_max_body_size    100m;
     }
 

configuration/nginx/http.conf

@@ -1,6 +1,6 @@
 # the upstream component nginx needs to connect to
 upstream django {
-    server uwsgi:8001 fail_timeout=30s;
+    server gunicorn:8001 fail_timeout=30s;
 }
 
 limit_req_zone $binary_remote_addr zone=adminlimit:10m rate=1r/s;

configuration/nginx/https.conf

@@ -1,6 +1,6 @@
 # the upstream component nginx needs to connect to
 upstream django {
-    server uwsgi:8001 fail_timeout=30s;
+    server gunicorn:8001 fail_timeout=30s;
 }
 
 server {

configuration/nginx/locations.conf

@@ -7,14 +7,17 @@ location /hc {
     return 200;
 }
 
-# All requests to the Django/UWSGI server.
+# All requests to the Django/Gunicorn server.
 location / {
     root /;
-    uwsgi_pass                  django;
-    uwsgi_pass_header           Authorization;
-    uwsgi_pass_request_headers  on;
-    uwsgi_read_timeout          600;
-    include                     uwsgi_params;
+    proxy_pass                  http://django;
+    proxy_set_header            Host $http_host;
+    proxy_set_header            X-Real-IP $remote_addr;
+    proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header            X-Forwarded-Proto $scheme;
+    proxy_set_header            X-Forwarded-Host $host;
+    proxy_redirect              off;
+    proxy_read_timeout          600;
     client_max_body_size        100m;
 }
 
@@ -23,11 +26,14 @@ location / {
 location ^~/admin {
     limit_req zone=adminlimit;
 
-    uwsgi_pass                  django;
-    uwsgi_pass_header           Authorization;
-    uwsgi_pass_request_headers  on;
-    uwsgi_read_timeout          600;
-    include                     uwsgi_params;
+    proxy_pass                  http://django;
+    proxy_set_header            Host $http_host;
+    proxy_set_header            X-Real-IP $remote_addr;
+    proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header            X-Forwarded-Proto $scheme;
+    proxy_set_header            X-Forwarded-Host $host;
+    proxy_redirect              off;
+    proxy_read_timeout          600;
     client_max_body_size        100m;
 }
 

docker/Dockerfile

@@ -30,7 +30,7 @@ SHELL ["/bin/bash", "-c"]
 
 RUN mkdir -p ${LOG_PATH} \
     ${LOG_PATH}/django \
-    ${LOG_PATH}/uwsgi \
+    ${LOG_PATH}/gunicorn \
     ${LOG_PATH}/asgi \
     /opt/deploy/files_required /opt/deploy/configuration /opt/deploy/files_required/blint /opt/deploy/files_required/yara
 
@@ -81,4 +81,7 @@ COPY --from=frontend-build /build /var/www/reactapp
 ENV HOME="${PYTHONPATH}"
 ENV XDG_CACHE_HOME="${PYTHONPATH}/.cache"
 
+# Default command to run Gunicorn
+CMD ["gunicorn", "intel_owl.wsgi:application", "-c", "/opt/deploy/intel_owl/configuration/gunicorn.conf.py"]
+
 

docker/ci.override.yml

@@ -8,7 +8,7 @@ services:
           cpus: '1'
           memory: 2000M
 
-  uwsgi:
+  gunicorn:
     build:
       context: ..
       dockerfile: docker/Dockerfile

docker/default.yml

@@ -3,24 +3,24 @@ x-no-healthcheck: &no-healthcheck
     disable: true
 
 services:
-  uwsgi:
+  gunicorn:
     image: intelowlproject/intelowl:${REACT_APP_INTELOWL_VERSION}
-    container_name: intelowl_uwsgi
+    container_name: intelowl_gunicorn
     volumes:
-      - ../configuration/intel_owl.ini:/etc/uwsgi/sites/intel_owl.ini # uwsgi configuration file
+      - ../configuration/gunicorn.conf.py:/opt/deploy/intel_owl/configuration/gunicorn.conf.py
       - ../configuration:/opt/deploy/intel_owl/configuration
       - generic_logs:/var/log/intel_owl
       - static_content:/opt/deploy/intel_owl/static
       - shared_files:/opt/deploy/files_required
     entrypoint:
-      - ./docker/entrypoints/uwsgi.sh
+      - ./docker/entrypoints/gunicorn.sh
     expose:
       - "8001"
     env_file:
       - env_file_app
       - .env
     healthcheck:
-      test: [ "CMD-SHELL", "nc -z localhost 8001 || exit 1" ]
+      test: [ "CMD-SHELL", "curl -f http://localhost:8001/hc || exit 1" ]
       interval: 5s
       timeout: 2s
       start_period: 300s
@@ -46,7 +46,7 @@ services:
       start_period: 2s
       retries: 6
     depends_on:
-      uwsgi:
+      gunicorn:
         condition: service_healthy
 
   nginx:
@@ -61,7 +61,7 @@ services:
       - nginx_logs:/var/log/nginx
       - static_content:/var/www/static
     depends_on:
-      uwsgi:
+      gunicorn:
         condition: service_healthy
       daphne:
         condition: service_healthy
@@ -80,7 +80,7 @@ services:
       - env_file_app
     <<: *no-healthcheck
     depends_on:
-      uwsgi:
+      gunicorn:
         condition: service_healthy
 
 
@@ -103,7 +103,7 @@ services:
     env_file:
       - env_file_app
     depends_on:
-      uwsgi:
+      gunicorn:
         condition: service_healthy
     <<: *no-healthcheck
 

docker/elasticsearch.override.yml

@@ -1,5 +1,5 @@
 services:
-  uwsgi:
+  gunicorn:
     depends_on:
       elasticsearch:
         condition: service_healthy

docker/entrypoints/uwsgi.sh → docker/entrypoints/gunicorn.sh

@@ -4,8 +4,8 @@ until cd /opt/deploy/intel_owl
 do
     echo "Waiting for server volume..."
 done
-mkdir -p /var/log/intel_owl/django /var/log/intel_owl/uwsgi /var/log/intel_owl/asgi /opt/deploy/files_required/blint /opt/deploy/files_required/yara
-chown -R www-data:www-data /var/log/intel_owl/django /var/log/intel_owl/uwsgi /var/log/intel_owl/asgi /opt/deploy/files_required/blint /opt/deploy/files_required/yara
+mkdir -p /var/log/intel_owl/django /var/log/intel_owl/gunicorn /var/log/intel_owl/asgi /opt/deploy/files_required/blint /opt/deploy/files_required/yara
+chown -R www-data:www-data /var/log/intel_owl/django /var/log/intel_owl/gunicorn /var/log/intel_owl/asgi /opt/deploy/files_required/blint /opt/deploy/files_required/yara
 
 # Apply database migrations
 echo "Waiting for db to be ready..."
@@ -49,5 +49,5 @@ then
 else
     $CHANGELOG_NOTIFICATION_COMMAND
     $ELASTIC_TEMPLATE_COMMAND
-    /usr/local/bin/uwsgi --ini /etc/uwsgi/sites/intel_owl.ini --stats 127.0.0.1:1717 --stats-http
+    exec gunicorn intel_owl.wsgi:application -c /opt/deploy/intel_owl/configuration/gunicorn.conf.py
 fi

docker/flower.override.yml

@@ -1,5 +1,5 @@
 services:
-  uwsgi:
+  gunicorn:
     environment:
       - BROKER_URL_API=http://guest:guest@rabbitmq:15672/api/
 
@@ -25,7 +25,7 @@ services:
       - "5555"
     depends_on:
       - rabbitmq
-      - uwsgi
+      - gunicorn
     entrypoint:
       - ./docker/entrypoints/flower.sh
     env_file:

docker/multi-queue.override.yml

@@ -3,7 +3,7 @@ x-no-healthcheck: &no-healthcheck
     disable: true
 
 services:
-  uwsgi:
+  gunicorn:
     environment:
       - CELERY_QUEUES=default,local,long
 
@@ -22,7 +22,7 @@ services:
     env_file:
       - env_file_app
     depends_on:
-      uwsgi:
+      gunicorn:
         condition: service_healthy
     <<: *no-healthcheck
 
@@ -41,7 +41,7 @@ services:
     env_file:
       - env_file_app
     depends_on:
-      uwsgi:
+      gunicorn:
         condition: service_healthy
     <<: *no-healthcheck
 
@@ -60,6 +60,6 @@ services:
     env_file:
       - env_file_app
     depends_on:
-      uwsgi:
+      gunicorn:
         condition: service_healthy
     <<: *no-healthcheck

docker/nfs.override.yml

@@ -1,5 +1,5 @@
 services:
-  uwsgi:
+  gunicorn:
     volumes:
       - nfs_files:/opt/deploy/files_required
     environment:

docker/postgres.override.yml

@@ -15,7 +15,7 @@ services:
       start_period: 3s
 
 
-  uwsgi:
+  gunicorn:
     depends_on:
       postgres:
         condition: service_healthy

docker/rabbitmq.override.yml

@@ -7,7 +7,7 @@ services:
     logging:
       driver: none
 
-  uwsgi:
+  gunicorn:
     environment:
       - BROKER_URL="amqp://guest:guest@rabbitmq:5672"
     depends_on:

docker/redis.override.yml

@@ -1,5 +1,5 @@
 services:
-  uwsgi:
+  gunicorn:
     depends_on:
       redis:
         condition: service_healthy

docker/scripts/initdb.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-docker exec intelowl_uwsgi python3 manage.py makemigrations
-docker exec intelowl_uwsgi python3 manage.py migrate
-docker exec -ti intelowl_uwsgi python3 manage.py createsuperuser \
+docker exec intelowl_gunicorn python3 manage.py makemigrations
+docker exec intelowl_gunicorn python3 manage.py migrate
+docker exec -ti intelowl_gunicorn python3 manage.py createsuperuser \
 --username admin --email admin@admin.com --first_name admin --last_name admin

docker/scripts/manage.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-docker exec -ti intelowl_uwsgi python3 manage.py "$@"
+docker exec -ti intelowl_gunicorn python3 manage.py "$@"

docker/scripts/tail-logs.sh

@@ -1,4 +1,4 @@
 #!/bin/bash  
 
-docker exec intelowl_uwsgi ls -al /var/log/intel_owl/"$1" 
-docker exec -ti intelowl_uwsgi tail -f /var/log/intel_owl/"$1"  
+docker exec intelowl_gunicorn ls -al /var/log/intel_owl/"$1" 
+docker exec -ti intelowl_gunicorn tail -f /var/log/intel_owl/"$1"  

docker/sqs.override.yml

@@ -1,5 +1,5 @@
 services:
-  uwsgi:
+  gunicorn:
     environment:
       - AWS_SQS=True
       - BROKER_URL=sqs://