HUB-687 Fix main

.env.example

@@ -47,8 +47,8 @@ CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
 # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
 CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
 CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
-CKAN_SYSADMIN_NAME=
-CKAN_SYSADMIN_PASSWORD=
+CKAN_SYSADMIN_NAME=CHANGE_ME
+CKAN_SYSADMIN_PASSWORD=CHANGE_ME
 CKAN_SYSADMIN_EMAIL=your_email@example.com
 CKAN_STORAGE_PATH=/var/lib/ckan
 CKAN_SMTP_SERVER=smtp.corporateict.domain:25
@@ -143,12 +143,19 @@ NEXT_PUBLIC_MAILING_LIST_EMAIL=ndp@sdsc.edu
 NEXT_PUBLIC_NODE_MAILER_EMAIL=
 NEXT_PUBLIC_NODE_MAILER_PSWD=
 NEXT_PUBLIC_ML_FLOW_URL=https://ndp-test.sdsc.edu/mlflow
-NEXT_PUBLIC_KEYCLOAK_API_URL=https://idp-test.nationaldataplatform.org/api
-NEXT_WORKSPACE_APIS_URL=https://ndp-test.sdsc.edu/workspaces-api
+NEXT_PUBLIC_KEYCLOAK_API_URL=https://idp.nationaldataplatform.org/temp
+NEXT_WORKSPACE_APIS_URL=http://workspaces-api:8000
 NEXT_PUBLIC_PREKAN_CATALOG_URL=https://ndp-test.sdsc.edu/catalog2/ndp
 NODE_ENV=production
 NEXT_PUBLIC_NDP_VERSION=v0.2 alpha version
 NEXT_PUBLIC_NDP_RELEASE_NOTES=https://github.com/national-data-platform/ndp/releases/tag/v0.2.0-alpha
+NEXT_PUBLIC_NDP_DOCS_LINK=https://ndp-test.sdsc.edu/documentation/
+NEXT_PUBLIC_KEYCLOAK_LOGOUT_URL=https://idp-test.nationaldataplatform.org/realms/NDP/protocol/openid-connect/logout
+NEXT_PUBLIC_POP_FEDERATION_URL=https://federation.ndp.utah.edu
+NEXT_PUBLIC_POP_FACTORY_URL=https://factory.ndp.utah.edu
+NEXT_PUBLIC_POP_TOKEN_URL=https://token.ndp.utah.edu
+NEXT_PUBLIC_DEMOCRATIZING_DATA_DASHBOARD_URL='https://appserver.democratizingdata.ai:5601/app/dashboards?auth_provider_hint=anonymous1#/view/77f3f14d-d308-44eb-ae44-114aaa83796e?embed=true&_g=(time%3A(from%3Anow-15m%2Cto%3Anow))&show-time-filter=true'
+NEXT_DATA_DEMOCRATIZATION_API=https://vdc-192.chpc.utah.edu/datainsight/api/
 
 #-------------------------
 ## ckanext-keycloak

.gitmodules

@@ -10,4 +10,6 @@
 [submodule "ndp-workspaces-api"]
 	path = ndp-workspaces-api
 	url = git@github.com:national-data-platform/ndp-workspaces-api.git
-	branch = main
+[submodule "ndp-documentation"]
+	path = ndp-documentation
+	url = git@github.com:national-data-platform/ndp-documentation.git

Makefile

@@ -9,13 +9,13 @@ build-run:
 	docker compose -f $(COMPOSE_FILE) up --build -d
 
 download-ckanext-ndp:
-	git -C ./src_extensions clone git@github.com:national-data-platform/ckanext-ndp.git
+	git -C ./src_extensions clone git@github.com:national-data-platform/ckanext-ndp.git --branch v0.1.0
 
 download-ckanext-keycloak:
-	git -C ./src_extensions clone git@github.com:national-data-platform/ckanext-keycloak.git --branch ndp
+	git -C ./src_extensions clone git@github.com:national-data-platform/ckanext-keycloak.git --branch v0.3.0
 
 download-ckanext-ndpcatalogadditions:
-	git -C ./src_extensions clone git@github.com:national-data-platform/ckanext-ndpcatalogadditions.git --branch 0.1.1
+	git -C ./src_extensions clone git@github.com:national-data-platform/ckanext-ndpcatalogadditions.git --branch main
 
 update-ckan-config:
 	docker compose -f $(COMPOSE_FILE) exec -it ckan /bin/bash -c "ckan config-tool /srv/app/ckan.ini ckanext.ndp.jupyterhub_endpoint=http://localhost:8000"

README.md

@@ -165,6 +165,7 @@ After this change you will need to restart ckan to pick up the new changes.
 ### Remarks
 1. There are cases when ckan doesn't pick up some environment variables from the .env file so sometimes its better to mount a copy of the ckan.ini file. There is a comment in the docker-compose file for this in this [line](/docker-compose.dev.yaml?plain=24). I have noticed that it sometimes doesn't pick up `CKAN___ROOT__PATH` which is needed for the nginx reverse proxy to work.
 2. Same issue, as #1. There are cases when ckan2 doesn't pick up some environment variables from the .env_ckan2 file so sometimes its better to mount a copy of the ckan2.ini file.
+3. Both instances of CKAN need the environement variables, `CKAN_SYSADMIN_NAME` and `CKAN_SYSADMIN_PASSWORD`,to be changed in order for the datapusher microservice to not error out.
 
 ## Production mode
 The production docker-compose file is docker-compose.prod.yaml.

docker-compose.dev.yaml

@@ -23,7 +23,7 @@ services:
       - ckan_storage:/var/lib/ckan
       - ./src_extensions:/srv/app/src_extensions
       - ./logs:/srv/app/logs
-      - ./supervisord/ckan_harvesting.conf:/etc/supervisord.d/ckan_harvesting.conf
+      - ./ckan-docker/supervisord/ckan_harvesting.conf:/etc/supervisord.d/ckan_harvesting.conf
       - ./ckan.ini:/srv/app/ckan.ini
     restart: always
     healthcheck:
@@ -68,20 +68,6 @@ services:
     restart: always
 
   # starting mlflow section
-  minio:
-    platform: linux/amd64
-    extends:
-      file: mlflow/docker-compose.yaml
-      service: minio
-    restart: always
-
-  create-s3-buckets:
-    platform: linux/amd64
-    extends:
-      file: mlflow/docker-compose.yaml
-      service: create-s3-buckets
-    restart: always
-
   mlflow:
     platform: linux/amd64
     build:
@@ -90,7 +76,6 @@ services:
     volumes:
       - "mlflow:/mlflow"
     depends_on:
-      - minio
       - db
     environment:
       - GUNICORN_CMD_ARGS
@@ -111,45 +96,15 @@ services:
       - MLFLOW_TRACKING_USERNAME
       - MLFLOW_TRACKING_PASSWORD
       - MLFLOW_ROOT_PATH
+    command: >
+      mlflow server
+        --host 0.0.0.0
+        --port ${MLFLOW_PORT}
+        --backend-store-uri postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
+        --default-artifact-root /mlflow/artifacts
+        --app-name ndp_mlflow_auth
     restart: always
 
-  # starting jhub section
-  # jupyterhub:
-  #   build:
-  #     context: jhub
-  #     dockerfile: Dockerfile
-  #     args:
-  #       JUPYTERHUB_VERSION: latest
-  #   restart: always
-  #   image: jupyterhub
-  #   volumes:
-  #     - "./jhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro"
-  #     - "/var/run/docker.sock:/var/run/docker.sock:rw"
-  #     - "jupyterhub:/data"
-  #   ports:
-  #     - ${JUPYTERHUB_PROXY_PORT}:${JUPYTERHUB_PROXY_PORT}
-  #   environment:
-  #     - JUPYTERHUB_PROXY_PORT
-  #     - JUPYTERHUB_PORT
-  #     - DOCKER_NETWORK_NAME
-  #     - DOCKER_NOTEBOOK_IMAGE
-  #     - DOCKER_NOTEBOOK_DIR
-  #     # for MLFlow connection:
-  #     - AWS_ACCESS_KEY_ID
-  #     - AWS_SECRET_ACCESS_KEY
-  #     - MLFLOW_TRACKING_URI
-  #     - MLFLOW_S3_ENDPOINT_URL
-  #     - AWS_BUCKET_NAME
-  #     - MLFLOW_DEFAULT_PASSWORD
-  #     - MLFLOW_ADMIN_USERNAME
-  #     - MLFLOW_ADMIN_PASSWORD
-  #     # keycloak
-  #     - OAUTH2_AUTHORIZE_URL
-  #     - OAUTH2_TOKEN_URL
-  #     - KEYCLOAK_USERDATA_URL
-  #     - JUPYTERHUB_KEYCLOAK_CLIENT_ID
-  #     - JUPYTERHUB_KEYCLOAK_CLIENT_SECRET
-
   frontend:
 #    platform: linux/amd64
     build:
@@ -177,8 +132,18 @@ services:
       - NEXT_PUBLIC_PREKAN_CATALOG_URL
       - NEXT_PUBLIC_NDP_VERSION
       - NEXT_PUBLIC_NDP_RELEASE_NOTES
+      - NEXT_PUBLIC_NDP_DOCS_LINK
+      - NEXT_PUBLIC_KEYCLOAK_LOGOUT_URL
+      - NEXT_PUBLIC_POP_FEDERATION_URL
+      - NEXT_PUBLIC_POP_FACTORY_URL
+      - NEXT_PUBLIC_POP_TOKEN_URL
+      - NEXT_PUBLIC_DEMOCRATIZING_DATA_DASHBOARD_URL
+      - NEXT_DATA_DEMOCRATIZATION_API
 
-
+    volumes:
+      - ./ndp-frontend:/app
+      - /app/node_modules
+    command: ["sh", "-c", "npm run build && npm run dev"]
 
   nginx:
     platform: linux/amd64
@@ -195,6 +160,7 @@ services:
       - CKAN_PORT
       - JUPYTERHUB_PROXY_PORT
       - MLFLOW_PORT
+      - ENV=dev
     volumes:
       - ./nginx/nginx.conf:/etc/nginx/nginx.conf
       - ./nginx/default.conf:/etc/nginx/conf.d/default.conf
@@ -206,44 +172,43 @@ services:
       dockerfile: Dockerfile
 #    ports:
 #      - "8005:8000"
-    restart: always
+    # command: ["sh", "-c", "uvicorn src.main:app --reload --workers 8 --host 0.0.0.0 --port 8000"]
+    restart: always  # This will restart the container if it fails or becomes unhealthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]  # Health check to the FastAPI app
+      interval: 30s  # Time between health checks (30 seconds)
+      timeout: 10s    # Time to wait for a health check response
+      retries: 3      # Number of retries before marking the container as "unhealthy"
+      start_period: 10s  # Time to wait before starting health checks after container starts
     depends_on:
       - workspace-db
     environment:
       - CKAN_SQLALCHEMY_URL
-      - server_url
-      - realm
-      - client_id
-      - client_secret
-      - authorization_url
-      - token_url
       - WFR_BASE_PATH
-      - web_protocol
-      - base_host
-      - root_url
-      - backend_port
-      - frontend_port
-      - db_name
-      - db_user
-      - db_password
-      - db_schema
-      - postgresql_jar_path
-      - backend_full_url
-      - download_tmp_dir
+      - ROOT_URL
+      - BACKEND_PORT
+      - WORKSPACE_DB_URL
       - WFR_DOCS_URL
+      - SERVER_URL
+      - REALM
+      - CLIENT_ID
+      - JUPYTERHUB_CLIENT_ID
+      - CLIENT_SECRET
+      - AUTHORIZATION_URL
+      - TOKEN_URL
+      - ADMIN_USERNAME
+      - ADMIN_PASSWORD
+      - ADMIN_REALM
+      - FROM_EMAIL
+      - EMAIL_SMTP
+      - EMAIL_PORT
+      - EMAIL_SECRET
+      - JUPYTERHUB_CLIENT_SECRET
+      - CKAN_DEFAULT
       - API_PROXY_PATH
-      - admin_username
-      - admin_password
-      - admin_realm
-      - AWESOME_URL
-      - from_email
-      - email_smtp
-      - email_port
-      - email_secret
-      - jupyterhub_client_id
-      - jupyterhub_client_secret
-  #    volumes:
-#        - ./ndp-workspaces-api/src:/code/src
+    volumes:
+        - ./ndp-workspaces-api/src:/code/src
+        - ./ndp-workspaces-api/alembic:/code/alembic
 
   # second instance of ckan (we call it Precan)
   ckan2:
@@ -268,7 +233,7 @@ services:
       - ./src_extensions:/srv/app/src_extensions
       - ./ckan2.ini:/srv/app/ckan.ini
       - ./logs:/srv/app/logs
-      - ./supervisord/ckan_harvesting.conf:/etc/supervisord.d/ckan_harvesting.conf
+      # - ./supervisord/ckan_harvesting.conf:/etc/supervisord.d/ckan_harvesting.conf
     restart: always
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
@@ -310,18 +275,29 @@ services:
     restart: always
 
   workspace-db:
-    image: postgres:latest
+    image: postgres:16.4-alpine
     environment:
       - POSTGRES_USER=${POSTGRES_USER_WORKSPACES}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD_WORKSPACES}
       - POSTGRES_DB=${POSTGRES_DB_WORKSPACES}
     volumes:
       - pg_workspace_data:/var/lib/postgresql/data
+      - ./ndp-workspaces-api/sql_scripts:/docker-entrypoint-initdb.d:ro
     restart: unless-stopped
       # ports:
       # - "8888:5432"
     healthcheck:
-      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER_WORKSPACES}", "-d", "${POSTGRES_DB_WORKSPACES}"]
+
+  mkdocs:
+    image: squidfunk/mkdocs-material
+#    ports:
+#      - "8000:8000"
+    volumes:
+      - ./ndp-documentation:/docs
+    stdin_open: true
+    tty: true
+    restart: always
 
 volumes:
   ckan_storage:
@@ -331,6 +307,4 @@ volumes:
   pg_workspace_data:
   solr_data:
   solr_data2:
-  miniodata:
   mlflow:
-#  jupyterhub:

docker-compose.prod.yaml

@@ -177,6 +177,7 @@ services:
       - NEXT_PUBLIC_PREKAN_CATALOG_URL
       - NEXT_PUBLIC_NDP_VERSION
       - NEXT_PUBLIC_NDP_RELEASE_NOTES
+      - NEXT_PUBLIC_NDP_DOCS_LINK
 
 
 
@@ -207,6 +208,14 @@ services:
 #    ports:
 #      - "8005:8000"
     restart: always
+#    command: ["sh", "-c", "uvicorn src.main:app --workers 8 --host 0.0.0.0 --port 8000"]
+    command: ["sh", "-c", "gunicorn -k uvicorn.workers.UvicornWorker src.main:app --workers 4 --bind 0.0.0.0:8000"]
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]  # Health check to the FastAPI app
+      interval: 30s  # Time between health checks (30 seconds)
+      timeout: 10s    # Time to wait for a health check response
+      retries: 3      # Number of retries before marking the container as "unhealthy"
+      start_period: 10s  # Time to wait before starting health checks after container starts
     depends_on:
       - workspace-db
     environment:
@@ -323,6 +332,16 @@ services:
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
 
+  mkdocs:
+    image: squidfunk/mkdocs-material
+#    ports:
+#      - "8000:8000"
+    volumes:
+      - ./ndp-documentation:/docs
+    stdin_open: true
+    tty: true
+    restart: always
+
 volumes:
   ckan_storage:
   ckan_storage2:

nginx/default.conf

@@ -4,11 +4,11 @@ map $http_upgrade $connection_upgrade {
 }
 
 server {
-    #listen       80;
-    #listen  [::]:80;
-    listen       443 default_server;
-    listen  [::]:443 default_server;
-    server_name  _;
+    listen       80;
+    listen  [::]:80;
+    # listen       443 default_server;
+    # listen  [::]:443 default_server;
+    server_name  localhost;
     charset utf-8;
 
     #access_log  /var/log/nginx/host.access.log  main;
@@ -37,6 +37,11 @@ server {
 
     location / {
         proxy_pass http://frontend:3000/;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_cache_bypass $http_upgrade;
     }
 
     location ^~ /ajax-api {
@@ -68,4 +73,8 @@ server {
     location /workspaces-api {
         proxy_pass http://workspaces-api:8000;
     }
+
+    location /documentation {
+        proxy_pass http://mkdocs:8000;
+    }
 }