v2.1.1 dev to master

.dockerignore

@@ -6,4 +6,6 @@ server/index
 server/keypair
 server/sqlite
 .git
-node_modules
+node_modules
+server/node_modules
+

.env

@@ -3,15 +3,14 @@
 DATABUS_RESOURCE_BASE_URL=http://localhost:3000
 # do not use qoutation marks for the value here and no trailing space
 DATABUS_NAME=My Local Databus
+DATABUS_ABSTRACT=A local instance for development
 DATABUS_ORG_ICON=
 DATABUS_BANNER_COLOR= # Default is #81b8b2
 DATABUS_TITLE_COLOR= # Default is white
 # OIDC SETTINGS - Only use these values for testing and enter your own OIDC provider values
 DATABUS_OIDC_ISSUER_BASE_URL=https://kilt.eu.auth0.com
 DATABUS_OIDC_CLIENT_ID=K5PCEOr7OJGBGU9xN7SvBrX1RWDs4S4n
 DATABUS_OIDC_SECRET=LiL_X1tzwRmReU3RO7kBlBdDopmVEGf4gj5Ve8No16kifyi3weXK7u6IS1Ttpl_q
-
-
 DATABUS_PRIVATE_MODE=false
 
 ####### DATABASE SETTINGS #######

.vscode/launch.json

@@ -0,0 +1,84 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Start Databus With AUTH0",
+      "program": "${workspaceFolder}/server/www",
+      "cwd": "${workspaceFolder}/server",
+      "runtimeArgs": [
+        "--trace-warnings"
+      ],
+      "env": {
+        "DATABUS_RESOURCE_BASE_URL": "http://localhost:3000",
+        "DATABUS_NAME": "Local Databus",
+        "DATABUS_ABSTRACT": "Go ham",
+        "DATABUS_ORG_ICON": "https://www.dbpedia.org/wp-content/uploads/2020/09/dbpedia-org-logo.png",
+        "DATABUS_BANNER_COLOR": "\\#5b798d",
+        "DATABUS_DATABASE_URL": "http://localhost:3002",
+        "DATABUS_OIDC_ISSUER_BASE_URL": "https://kilt.eu.auth0.com",
+        "DATABUS_OIDC_CLIENT_ID": "e9eOLS9IkGvuyBl7cBorWUQTNgbqejhc",
+        "DATABUS_OIDC_REQUIRED_ROLE": "",
+        "DATABUS_OIDC_SECRET": "0RVo9jMnbhPnkR6Ttz0aXQRTcRuSz5DpqyUEjbbcbgRuGA4rbwCjnHM2cOlTrv9q",
+        "LOOKUP_BASE_URL": "http://localhost:3004",
+        "DATABUS_OIDC_USER_ID_PROPERTY": "sub",
+        "MAX_WORKERS": "1",
+        "METRICS_PORT": "3001"
+      }
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Start Databus With DBpedia Community Account",
+      "program": "${workspaceFolder}/server/www",
+      "cwd": "${workspaceFolder}/server",
+      "runtimeArgs": [
+        "--trace-warnings"
+      ],
+      "env": {
+        "DATABUS_RESOURCE_BASE_URL": "http://localhost:3000",
+        "DATABUS_NAME": "Local Databus",
+        "DATABUS_ABSTRACT": "Go ham",
+        "DATABUS_ORG_ICON": "https://www.dbpedia.org/wp-content/uploads/2020/09/dbpedia-org-logo.png",
+        "DATABUS_BANNER_COLOR": "\\#5b798d",
+        "DATABUS_DATABASE_URL": "http://localhost:3002",
+        "DATABUS_OIDC_ISSUER_BASE_URL": "https://auth.dbpedia.org/realms/dbpedia",
+        "DATABUS_OIDC_CLIENT_ID": "databus-dev",
+        "DATABUS_OIDC_SECRET": "O55igP05y7iMlZdhweAEUjjhBIkiu0Mq",
+        "DATABUS_OIDC_USER_ID_PROPERTY": "sub",
+        "DATABUS_OIDC_REQUIRED_ROLE": "",
+        "LOOKUP_BASE_URL": "http://localhost:3004",
+        "MAX_WORKERS": "1",
+        "METRICS_PORT": "3001"
+      }
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Run Tests",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": [
+        "test"
+      ],
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}/server",
+      "env": {
+        "DATABUS_RESOURCE_BASE_URL": "http://localhost:3000",
+        "DATABUS_NAME": "Local Databus",
+        "DATABUS_ABSTRACT": "Go ham",
+        "DATABUS_ORG_ICON": "https://www.dbpedia.org/wp-content/uploads/2020/09/dbpedia-org-logo.png",
+        "DATABUS_BANNER_COLOR": "#5b798d",
+        "DATABUS_DATABASE_URL": "http://localhost:3002",
+        "DATABUS_OIDC_ISSUER_BASE_URL": "https://auth.dbpedia.org/realms/dbpedia",
+        "DATABUS_OIDC_CLIENT_ID": "databus-dev",
+        "DATABUS_OIDC_SECRET": "O55igP05y7iMlZdhweAEUjjhBIkiu0Mq",
+        "DATABUS_OIDC_REQUIRED_ROLE": "",
+        "DATABUS_OIDC_USER_ID_PROPERTY": "sub",
+        "LOOKUP_BASE_URL": "http://localhost:3004",
+        "MAX_WORKERS": "1",
+        "METRICS_PORT": "3001"
+      }
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "editor.detectIndentation": true
+}

Dockerfile

@@ -1,56 +1,50 @@
+<<<<<<< HEAD
+# Use a consistent Node.js version across build and release stages
+FROM node:23-slim AS builder
+=======
 FROM docker.io/ubuntu:22.04
+>>>>>>> master
 
-# Install node.js, Caddy as proxy server, and java.
-RUN apt-get update && \
-    apt-get -y install curl debian-keyring debian-archive-keyring apt-transport-https && \
-    curl -sL https://deb.nodesource.com/setup_16.x | bash - && \
-    curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg && \
-    curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list && \
-    apt-get update && \
-    apt-get -y install \
-      nodejs \
-      caddy \
-      ca-certificates-java \
-      openjdk-17-jdk \
-      openjdk-17-jre && \
-    rm -rf /var/lib/apt/lists/* && \
-    node -v && \
-    npm -v && \
-    java -version
-
-# Disable the proxy server by default:
-ENV DATABUS_PROXY_SERVER_ENABLE=false
-
-# When using the proxy, use ACME by default:
-ENV DATABUS_PROXY_SERVER_USE_ACME=true
-
-# When not using ACME, what is the name of the own certificate file?
-ENV DATABUS_PROXY_SERVER_OWN_CERT="cert.pem"
-
-# When not using ACME, what is the name of the own certificate's key file?
-ENV DATABUS_PROXY_SERVER_OWN_CERT_KEY="key.pem"
-
-# What is the hostname of this machine, when using the proxy server?
-# It is necessary to know this, in order to set up ACME etc.
-# Note: the host name should be identical to DATABUS_RESOURCE_BASE_URL,
-# but without specifying a port, protocol i.e. HTTP(S) etc.
-ENV DATABUS_PROXY_SERVER_HOSTNAME="my-databus.org"
-
-# Define the volume for the TLS certificate:
-VOLUME /tls
-
-COPY ./server /databus/server
-COPY ./public /databus/public
-COPY ./search /databus/search
-COPY ./model/generated /databus/model/generated
-
-COPY ./setup.sh /databus/setup.sh
-
-# Set up the NPM projects:
-RUN cd /databus/server && \
-    npm install && \
-    cd ../public && \
-    npm install
+# Install Python for node-gyp (required for sqlite3 build)
+RUN apt-get update && apt-get install -y python3 build-essential && \
+    ln -s /usr/bin/python3 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /databus/server
+COPY server/package*.json . 
+RUN --mount=type=bind,source=server/package.json,target=package.json \
+    --mount=type=cache,target=/root/.npm \
+    bash -c "if [ ! -f package-lock.json ]; then npm install --package-lock-only; fi && npm ci --omit=dev"
+
+WORKDIR /databus/public
+COPY public/package*.json .
+RUN --mount=type=bind,source=public/package.json,target=package.json \
+    --mount=type=cache,target=/root/.npm \
+    bash -c "if [ ! -f package-lock.json ]; then npm install --package-lock-only; fi && npm ci --omit=dev"
+
+WORKDIR /
+
+# Copy source code
+COPY server ./databus/server
+COPY public ./databus/public
+COPY search ./databus/search
+
+# Rebuild sqlite3 in case it's a native module
+# RUN cd databus/server && npm rebuild sqlite3
+
+FROM node:23-slim AS runtime
 
 WORKDIR /databus
-ENTRYPOINT [ "/bin/bash", "./setup.sh" ]
+
+# Install only necessary system packages
+RUN apt-get update && \
+    apt-get install -y \
+        curl \
+        ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy app from build stage
+COPY --from=builder /databus .
+
+# Entrypoint
+ENTRYPOINT [ "/bin/bash", "./server/setup.sh" ]

backup.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# VARIABLES
+
+# Define a prefix for the backup filename
+prefix="backup"
+
+# Get the current date in YYYY-MM-DD format
+date=$(date +%F)
+
+# Compose the full filename with prefix and date
+filename="${prefix}_${date}.tar.gz"
+
+# Specify the local folder to compress
+localFolder="/devenv/data"
+
+# Target folder on the remote server to copy the file into
+targetFolder="/path/to/remote/folder"
+
+# Shut down the databus, adjust for stack
+docker compose stop
+
+# Create a tar.gz archive of the local folder with the composed filename
+tar -czf "$filename" "$localFolder"
+
+# Securely copy the archive to the remote server's target folder
+scp "$filename" "$targetFolder"
+
+# Remove the local tar.gz archive to clean up
+rm "$filename"
+
+# Restart the databus, adjust for stack
+docker compose up -d

deployment/.env

@@ -0,0 +1,20 @@
+####### DATABUS SETTINGS #######
+DATA_PATH="../../databus-data"
+# RESOURCE BASE URL - This should match the DNS entry pointing to your server
+DATABUS_RESOURCE_BASE_URL=http://localhost:3000
+# do not use qoutation marks for the value here and no trailing space
+DATABUS_NAME=My Local Databus
+DATABUS_ABSTRACT=A local instance for development
+DATABUS_ORG_ICON=
+DATABUS_BANNER_COLOR= # Default is #81b8b2
+DATABUS_TITLE_COLOR= # Default is white
+# OIDC SETTINGS - Only use these values for testing and enter your own OIDC provider values
+DATABUS_OIDC_ISSUER_BASE_URL=https://kilt.eu.auth0.com
+DATABUS_OIDC_CLIENT_ID=K5PCEOr7OJGBGU9xN7SvBrX1RWDs4S4n
+DATABUS_OIDC_SECRET=LiL_X1tzwRmReU3RO7kBlBdDopmVEGf4gj5Ve8No16kifyi3weXK7u6IS1Ttpl_q
+DATABUS_PRIVATE_MODE=false
+
+####### DATABASE SETTINGS #######
+VIRTUOSO_USER=dba
+VIRTUOSO_PASSWORD=onlyweknow
+

deployment/README.md

@@ -0,0 +1,10 @@
+
+# Install Instructions
+
+## Starting a local Databus
+
+To start a local Databus instance
+
+```
+docker compose up
+```

deployment/docker-compose.yml

@@ -0,0 +1,59 @@
+name: "databus-dockerized"
+services:
+  databus:
+    image: "docker.io/dbpedia/databus:dev"
+    ports:
+      - 3000:3000 
+      - 3001:3001
+    environment:
+      DATABUS_RESOURCE_BASE_URL: ${DATABUS_RESOURCE_BASE_URL}
+      DATABUS_DATABASE_URL: "http://gstore:8080"
+      LOOKUP_BASE_URL: "http://lookup:8082"
+      METRICS_PORT: "3001"
+      DATABUS_OIDC_ISSUER_BASE_URL: ${DATABUS_OIDC_ISSUER_BASE_URL}
+      DATABUS_NAME: ${DATABUS_NAME}
+      DATABUS_ABSTRACT: ${DATABUS_ABSTRACT}
+      DATABUS_ORG_ICON: ${DATABUS_ORG_ICON}
+      DATABUS_BANNER_COLOR: ${DATABUS_BANNER_COLOR}
+      DATABUS_OIDC_CLIENT_ID: ${DATABUS_OIDC_CLIENT_ID}
+      DATABUS_OIDC_SECRET: ${DATABUS_OIDC_SECRET}
+      DATABUS_PRIVATE_MODE: ${DATABUS_PRIVATE_MODE} # Private mode will only show metadata for logged-in users
+    volumes:
+      - ${DATA_PATH}/data/dav/:/databus/server/dav/
+      - ${DATA_PATH}/data/sqlite/:/databus/server/sqlite
+      - ${DATA_PATH}/data/keypair/:/databus/server/keypair                                                                 
+  gstore:
+    image: "docker.io/dbpedia/gstore"
+    environment:
+      STORAGE_USER: ${VIRTUOSO_USER}
+      STORAGE_PASS:  ${VIRTUOSO_PASSWORD}
+      STORAGE_SPARQL_ENDPOINT_URI: http://virtuoso:8890/sparql
+      DEFAULT_JSONLD_LOCALHOST_CONTEXT: http://localhost:3000/res/context.jsonld
+      DEFAULT_JSONLD_LOCALHOST_CONTEXT_LOCATION: http://databus:3000/res/context.jsonld
+    ports:
+      - 3002:8080
+    volumes:
+      - ${DATA_PATH}/data/gstore/repo:/gstore/git
+      - ${DATA_PATH}/data/gstore/logs:/gstore/logs
+  virtuoso:
+    image: "docker.io/openlink/virtuoso-opensource-7"
+    environment:
+      DBA_PASSWORD: ${VIRTUOSO_PASSWORD}
+      SPARQL_UPDATE: "true"
+      DEFAULT_GRAPH: ${DATABUS_RESOURCE_BASE_URL}
+    ports:
+      - 3003:8890
+    volumes:
+      - ${DATA_PATH}/data/virtuoso:/database
+    entrypoint: /bin/bash -c "
+        echo 'grant SPARQL_LOAD_SERVICE_DATA to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
+        echo 'grant SPARQL_SPONGE to \"SPARQL\";' >> /opt/virtuoso-opensource/initdb.d/ini.sql &&
+        echo 'grant SPARQL_SELECT_FED to \"SPARQL\";' > /opt/virtuoso-opensource/initdb.d/ini.sql &&
+        /virtuoso-entrypoint.sh"
+  lookup:
+    image: "docker.io/dbpedia/lookup:dev"
+    ports:
+      - 3004:8082
+    volumes:
+      - ${DATA_PATH}/data/index/:/index
+      - ../search/servlet-config.yml:/resources/config.yml

devenv/Makefile

@@ -1,6 +1,7 @@
 DATABUS_RESOURCE_BASE_URL=http://localhost:3000
 DATABUS_DATABASE_URL=http://localhost:3002
 DATABUS_NAME="Just Another Databus"
+DATABUS_ABSTRACT="Another Node in the Databus Network"
 DATABUS_ORG_ICON="https://www.dbpedia.org/wp-content/uploads/2020/09/dbpedia-org-logo.png"
 DATABUS_BANNER_COLOR="\#5b798d"
 
@@ -9,7 +10,7 @@ webpack:
 
 env-clean:
 	sudo rm -r data/; \
-	sudo rm -r ../server/users; \
+	sudo rm -r ../server/users	
 	docker rm -f devenv_lookup; \
 	docker rm -f devenv_virtuoso; \
 	docker rm -f devenv_gstore;
@@ -42,7 +43,9 @@ srv-install:
 srv-start-auth0:
 	DATABUS_RESOURCE_BASE_URL=${DATABUS_RESOURCE_BASE_URL} \
 	DATABUS_NAME=${DATABUS_NAME} \
+	DATABUS_ABSTRACT=${DATABUS_ABSTRACT} \
 	DATABUS_ORG_ICON=${DATABUS_ORG_ICON} \
+	DATABUS_BANNER=${DATABUS_BANNER} \
 	DATABUS_BANNER_COLOR=${DATABUS_BANNER_COLOR} \
 	DATABUS_DATABASE_URL=${DATABUS_DATABASE_URL} \
 	DATABUS_OIDC_ISSUER_BASE_URL=https://kilt.eu.auth0.com \
@@ -81,3 +84,6 @@ srv-test:
 	DATABUS_OIDC_CLIENT_ID=e9eOLS9IkGvuyBl7cBorWUQTNgbqejhc \
 	DATABUS_OIDC_SECRET=0RVo9jMnbhPnkR6Ttz0aXQRTcRuSz5DpqyUEjbbcbgRuGA4rbwCjnHM2cOlTrv9q \
 	npm run test --prefix ./../server
+
+dockerize:
+	docker buildx build --push -t dbpedia/databus:dev -f ../Dockerfile ..

devenv/README.md

@@ -1,5 +1,12 @@
 # Development Environment
 
+## Orientation
+
+The Databus is implemented in Javascript using Nodejs+Express for the Server and Angularjs for the client-side webapp.
+The nodejs server is providing both the API and the webapp resources. The webapp does not have its own server and runs only in the browser.
+
+The Databus is best developed using Visual Studio Code to use the startup scripts in the `.vscode` folder.
+
 ## Requirements
 
 * `node.js`: v16.13.0 or higher