From 6810a46b373acadddb688401489829d02ab9a82c Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Sun, 5 Jan 2025 16:57:33 +0100 Subject: [PATCH 01/10] WIP Docker support --- InfoLogger/.dockerignore | 11 ++ InfoLogger/Dockerfile | 106 ++++++++++++++++++ InfoLogger/INFOLOGGER.sql | 87 ++++++++++++++ InfoLogger/README.md | 7 ++ InfoLogger/config-default.js | 2 +- InfoLogger/docker-compose.dev.yml | 63 +++++++++++ InfoLogger/docker-compose.yaml | 11 ++ .../test/live-simulator/infoLoggerServer.js | 22 ++-- 8 files changed, 297 insertions(+), 12 deletions(-) create mode 100644 InfoLogger/.dockerignore create mode 100644 InfoLogger/Dockerfile create mode 100644 InfoLogger/INFOLOGGER.sql create mode 100644 InfoLogger/docker-compose.dev.yml create mode 100644 InfoLogger/docker-compose.yaml diff --git a/InfoLogger/.dockerignore b/InfoLogger/.dockerignore new file mode 100644 index 000000000..14eed0154 --- /dev/null +++ b/InfoLogger/.dockerignore @@ -0,0 +1,11 @@ +# Default ignore everything +* + +# Project +lib/ +!package*.json +node_modules +public +index.js +!test +config.js \ No newline at end of file diff --git a/InfoLogger/Dockerfile b/InfoLogger/Dockerfile new file mode 100644 index 000000000..f26d9ef48 --- /dev/null +++ b/InfoLogger/Dockerfile @@ -0,0 +1,106 @@ +# +# ---- Base ---- +FROM node:22-alpine3.21 AS base + +RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app + +# Create app directory +WORKDIR /home/node/app + +# +# ---- Development Dependencies ---- +FROM base AS developmentdependencies + +# Installs Git and packages required for Puppeteer +# https://pkgs.alpinelinux.org/packages +RUN apk add --no-cache \ + chromium \ + freetype \ + freetype-dev + +# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + +# Copy all files, except those ignored by .dockerignore, to the container +COPY . . + + + +# Installs modules from package-lock.json if there are changes, this ensures reproducible build +RUN npm --silent ci + +# Install nodemon (for development only) +RUN npm i nodemon + +# +# ---- Development ---- +FROM developmentdependencies AS development + +# Expose the port to the Docker instance (not the host!) +EXPOSE 8080 + +# Run start script as specified in package.json +CMD [ "npm", "run", "dev" ] + + # # +# ---- Test ---- +FROM developmentdependencies AS test + +# Run start script as specified in package.json +CMD [ "npm", "run", "test" ] + + # # + # # ---- Test parallel for CI ---- + # FROM developmentdependencies AS test_parallel_ci + + # CMD [ "sh", "-c", "/opt/wait-for-it.sh -t 0 test_db:3306 -- npm run test:subset" ] + + # # + # # ---- Test parallel local ---- + # FROM developmentdependencies AS test_parallel_local + + # CMD [ "sh", "-c", "/opt/wait-for-it.sh -t 0 test_db:3306 -- npm run test:subset-local" ] + + # # + # # ---- Coverage ---- + # FROM developmentdependencies AS coverage + + # # Run start script as specified in package.json + # CMD [ "/opt/wait-for-it.sh", "-t", "0", "database:3306", "--", "npm", "run", "coverage" ] + + +# ---- Production Dependencies ---- +FROM base AS productiondependencies + +# Copy package.json and package-lock.json to the container. +COPY package*.json ./ + +# Installs modules from package-lock.json, this ensures reproducible build. +RUN npm --silent ci --production + +# Install nodemon (for development only) +RUN npm i nodemon + + # Copy all files, filtered by .dockerignore, to the container with added dev dependancies. +COPY . . +RUN rm -r ./test + + +# ---- Simul ---- +FROM developmentdependencies AS simul + +# Expose simul server port. +EXPOSE 6102 + +# Run simul script as specified in package.json +CMD [ "node", "run", "simul" ] + +# ---- Production ---- +FROM productiondependencies AS production + +# Expose the port to the Docker instance (not the host!) +EXPOSE 8080 + +# Run start script as specified in package.json +CMD [ "node", "run", "start" ] \ No newline at end of file diff --git a/InfoLogger/INFOLOGGER.sql b/InfoLogger/INFOLOGGER.sql new file mode 100644 index 000000000..4b33964d9 --- /dev/null +++ b/InfoLogger/INFOLOGGER.sql @@ -0,0 +1,87 @@ +-- phpMyAdmin SQL Dump +-- version 5.2.1 +-- https://www.phpmyadmin.net/ +-- +-- Host: mariadb +-- Generation Time: Dec 15, 2024 at 02:28 PM +-- Server version: 11.5.2-MariaDB-ubu2404 +-- PHP Version: 8.2.25 + +SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; +START TRANSACTION; +SET time_zone = "+00:00"; + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; + +-- +-- Database: `INFOLOGGER` +-- +CREATE DATABASE IF NOT EXISTS `INFOLOGGER` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_uca1400_ai_ci; +USE `INFOLOGGER`; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `messages` +-- + +CREATE TABLE `messages` ( + `severity` char(1) DEFAULT NULL, + `level` tinyint(3) UNSIGNED DEFAULT NULL, + `timestamp` double(16,6) DEFAULT NULL, + `hostname` varchar(32) DEFAULT NULL, + `rolename` varchar(32) DEFAULT NULL, + `pid` smallint(5) UNSIGNED DEFAULT NULL, + `username` varchar(32) DEFAULT NULL, + `system` varchar(32) DEFAULT NULL, + `facility` varchar(32) DEFAULT NULL, + `detector` varchar(32) DEFAULT NULL, + `partition` varchar(32) DEFAULT NULL, + `dest` varchar(32) DEFAULT NULL, + `run` int(10) UNSIGNED DEFAULT NULL, + `errcode` int(10) UNSIGNED DEFAULT NULL, + `errline` smallint(5) UNSIGNED DEFAULT NULL, + `errsource` varchar(32) DEFAULT NULL, + `message` text DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +PARTITION BY HASH (`timestamp` DIV 86400) +PARTITIONS 365; + +-- +-- Dumping data for table `messages` +-- + +INSERT INTO `messages` (`severity`, `level`, `timestamp`, `hostname`, `rolename`, `pid`, `username`, `system`, `facility`, `detector`, `partition`, `dest`, `run`, `errcode`, `errline`, `errsource`, `message`) VALUES +('I', 6, 1456134840.001504, 'aldaqecs01-v1', NULL, 2733, 'alicedaq', 'DAQ', 'LHCBeamInfo', NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'Mon Feb 22 10:54:00 2016 (1456134840): BeamMode: NO BEAM; BeamType: ; ParticleTypeB1: ; ParticleTypeB2: ; BetaStar: ; Eng: 0 GeV; FillN: -999; FillScheme: ; NbunchesInt: 0; NbunchesNotIntB1: 0; NbunchesNotIntB2: 0; InteracIst I(B1): 0; InteracTot I(B1): 0; InteracIst I(B2): 0; InteracTot I(B2): 0; NotInteracIst I(B1): 0; NotInteracTot I(B1): 0; NotInteracIst I(B2): 0; NotInteracTot I(B2): 0; PostMortemN: 1; LHCAdjust: STANDBY; LHCBeamDump: STANDBY; LHCInjection: STANDBY; DIPconnected Flag: 1 1 1 1 1 1 1 1 1 1 1 1 1 1'); + +-- +-- Indexes for dumped tables +-- + +-- +-- Indexes for table `messages` +-- +ALTER TABLE `messages` + ADD KEY `ix_severity` (`severity`), + ADD KEY `ix_level` (`level`), + ADD KEY `ix_timestamp` (`timestamp`), + ADD KEY `ix_hostname` (`hostname`(14)), + ADD KEY `ix_rolename` (`rolename`(20)), + ADD KEY `ix_system` (`system`(3)), + ADD KEY `ix_facility` (`facility`(20)), + ADD KEY `ix_detector` (`detector`(8)), + ADD KEY `ix_partition` (`partition`(10)), + ADD KEY `ix_dest` (`dest`(10)), + ADD KEY `ix_run` (`run`), + ADD KEY `ix_errcode` (`errcode`), + ADD KEY `ix_errline` (`errline`), + ADD KEY `ix_errsource` (`errsource`(20)); +COMMIT; + +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; diff --git a/InfoLogger/README.md b/InfoLogger/README.md index 9fdf37be3..b0ded6f93 100644 --- a/InfoLogger/README.md +++ b/InfoLogger/README.md @@ -90,6 +90,13 @@ mysql: { ``` 8. Run the InfoLogger and check for the following message in the console: `info: Connection to DB successfully established: 127.0.0.1:3306` +## Docker development + +docker compose -f docker-compose.yaml -f docker-compose.dev.yml down -v +docker compose -f docker-compose.yaml -f docker-compose.dev.yml up -d --build + +process.env.mariadb_host ?? '127.0.0.1' + ## Dummy InfoLogger test server InfoLoggerServer can be simulated by running `npm run simul`. The dummy server binds `localhost:6102` endpoint. ## InfoLogger insights diff --git a/InfoLogger/config-default.js b/InfoLogger/config-default.js index fcba3af58..0b65ce093 100644 --- a/InfoLogger/config-default.js +++ b/InfoLogger/config-default.js @@ -28,7 +28,7 @@ module.exports = { // optional data source, comment object if not used // all options: https://github.com/mysqljs/mysql#connection-options mysql: { - host: '127.0.0.1', + host: process.env.mariadb_host ?? '127.0.0.1', user: 'root', password: 'root', database: 'INFOLOGGER', diff --git a/InfoLogger/docker-compose.dev.yml b/InfoLogger/docker-compose.dev.yml new file mode 100644 index 000000000..d79f5cc4c --- /dev/null +++ b/InfoLogger/docker-compose.dev.yml @@ -0,0 +1,63 @@ +services: + application: + build: + target: development + environment: + NODE_ENV: development + mariadb_host: database + ports: + - "8080:8080" + command: "npm run dev" + restart: "no" + volumes: + - type: bind + read_only: true + source: ./lib + target: /home/node/app/lib + - type: bind + read_only: true + source: ./public + target: /home/node/app/public + - type: bind + read_only: true + source: ./test + target: /home/node/app/test + - type: bind + read_only: true + source: ./config.js + target: /home/node/app/config.js + - type: bind + read_only: true + source: ./index.js + target: /home/node/app/index.js + + database: + environment: + MARIADB_ROOT_PASSWORD: root + MARIADB_DATABASE: + ports: + - "3306:3306" + volumes: + - type: bind + read_only: true + source: ./INFOLOGGER.sql + target: /docker-entrypoint-initdb.d/INFOLOGGER.sql + phpmyadmin: + image: phpmyadmin + restart: unless-stopped + ports: + - 9090:80 + environment: + - PMA_HOST=database + simulator: + build: + target: simul + environment: + NODE_ENV: development + ports: + - "6102:6102" + # command: "npm run dev" + restart: "unless-stopped" + +volumes: + database-data: \ No newline at end of file diff --git a/InfoLogger/docker-compose.yaml b/InfoLogger/docker-compose.yaml new file mode 100644 index 000000000..4ae570012 --- /dev/null +++ b/InfoLogger/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + application: + build: + context: . + dockerfile: Dockerfile + target: production + database: + image: mariadb:latest + restart: unless-stopped + environment: + MARIADB_ROOT_PASSWORD: root diff --git a/InfoLogger/test/live-simulator/infoLoggerServer.js b/InfoLogger/test/live-simulator/infoLoggerServer.js index 1b736b888..1f15b7a22 100644 --- a/InfoLogger/test/live-simulator/infoLoggerServer.js +++ b/InfoLogger/test/live-simulator/infoLoggerServer.js @@ -10,10 +10,10 @@ * In applying this license CERN does not waive the privileges and immunities * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. -*/ + */ /* eslint-disable no-console */ -/* eslint-disable require-jsdoc */ +/* eslint-disable jsdoc/require-jsdoc */ // Documentation: // https://nodejs.org/api/net.html#net_net_createserver_options_connectionlistener @@ -36,12 +36,12 @@ const createServer = () => { function sendNextLog() { const log = fakeData[currentLogIndex % fakeData.length]; - const timestamp = (new Date()).getTime() / 1000; // seconds - const nextLogTimeout = 100 - (Math.random() * 100); // [0 ; 500]ms + const timestamp = new Date().getTime() / 1000; // seconds + const nextLogTimeout = 100 - Math.random() * 100; // [0 ; 500]ms // switch protocol after each log sent to try both protocols if (currentLogIndex % 2 === 1) { - client.write(`*1.4#` + + client.write('*1.4#' + `${log.severity || ''}#` + `${log.level || ''}#` + `${timestamp || ''}#` + @@ -59,7 +59,7 @@ const createServer = () => { `${log.errsource || ''}#` + `${log.message || ''}\r\n`); } else { - client.write(`*1.3#` + + client.write('*1.3#' + `${log.severity || ''}#` + `${log.level || ''}#` + `${timestamp || ''}#` + @@ -71,7 +71,7 @@ const createServer = () => { `${log.facility || ''}#` + `${log.detector || ''}#` + `${log.partition || ''}#` + - `#` + // dest field + '#' + // dest field `${log.run || ''}#` + `${log.errcode || ''}#` + `${log.errline || ''}#` + @@ -97,8 +97,8 @@ const createServer = () => { server.listen(port, () => { console.log(`InfoLoggerServer is running on port ${port}`); }); - return server -} + return server; +}; const closeServer = (server) => { try { @@ -106,6 +106,6 @@ const closeServer = (server) => { } catch (err) { console.error(err); } -} +}; -module.exports = {createServer, closeServer}; \ No newline at end of file +module.exports = { createServer, closeServer }; From 62a2d5df0800f6ab2b501a82a498a44eda600fdd Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Sun, 5 Jan 2025 22:40:38 +0100 Subject: [PATCH 02/10] Docker dev deployment, works with nodemon. --- InfoLogger/config-default.js | 2 +- InfoLogger/docker-compose.dev.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/InfoLogger/config-default.js b/InfoLogger/config-default.js index 0b65ce093..6b99a26b5 100644 --- a/InfoLogger/config-default.js +++ b/InfoLogger/config-default.js @@ -40,7 +40,7 @@ module.exports = { // optional data source, comment object if not used // all options: https://nodejs.org/api/net.html#net_socket_connect_options_connectlistener infoLoggerServer: { - host: 'localhost', + host: process.env.infologger_host ?? 'localhost', port: 6102, }, logging: { diff --git a/InfoLogger/docker-compose.dev.yml b/InfoLogger/docker-compose.dev.yml index d79f5cc4c..65b76d6ed 100644 --- a/InfoLogger/docker-compose.dev.yml +++ b/InfoLogger/docker-compose.dev.yml @@ -5,6 +5,7 @@ services: environment: NODE_ENV: development mariadb_host: database + infologger_host: simulator ports: - "8080:8080" command: "npm run dev" @@ -56,7 +57,7 @@ services: NODE_ENV: development ports: - "6102:6102" - # command: "npm run dev" + command: "npm run simul" restart: "unless-stopped" volumes: From cfb924b0c786097e08854ebbbf7b2eca755e88f5 Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Sun, 5 Jan 2025 22:48:58 +0100 Subject: [PATCH 03/10] DB health check + container order/dependencies changed --- InfoLogger/docker-compose.dev.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/InfoLogger/docker-compose.dev.yml b/InfoLogger/docker-compose.dev.yml index 65b76d6ed..27b044290 100644 --- a/InfoLogger/docker-compose.dev.yml +++ b/InfoLogger/docker-compose.dev.yml @@ -2,6 +2,10 @@ services: application: build: target: development + depends_on: + database: + condition: service_healthy + restart: true environment: NODE_ENV: development mariadb_host: database @@ -43,9 +47,18 @@ services: read_only: true source: ./INFOLOGGER.sql target: /docker-entrypoint-initdb.d/INFOLOGGER.sql + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 1s + timeout: 20s + retries: 20 phpmyadmin: image: phpmyadmin restart: unless-stopped + depends_on: + database: + condition: service_healthy + restart: true ports: - 9090:80 environment: From 24d1a7e44654f933b9101328744b7b696981471e Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Mon, 6 Jan 2025 17:44:08 +0100 Subject: [PATCH 04/10] Docker support finalized. --- InfoLogger/.dockerignore | 10 +++---- InfoLogger/Dockerfile | 28 ++----------------- InfoLogger/README.md | 9 ++++-- ...ompose.dev.yml => docker-compose.dev.yaml} | 20 ++++++------- InfoLogger/docker-compose.simul.yaml | 9 ++++++ InfoLogger/docker-compose.test.yaml | 28 +++++++++++++++++++ InfoLogger/docker-compose.yaml | 7 +---- InfoLogger/package.json | 4 +++ 8 files changed, 65 insertions(+), 50 deletions(-) rename InfoLogger/{docker-compose.dev.yml => docker-compose.dev.yaml} (87%) create mode 100644 InfoLogger/docker-compose.simul.yaml create mode 100644 InfoLogger/docker-compose.test.yaml diff --git a/InfoLogger/.dockerignore b/InfoLogger/.dockerignore index 14eed0154..b1a793ffd 100644 --- a/InfoLogger/.dockerignore +++ b/InfoLogger/.dockerignore @@ -2,10 +2,10 @@ * # Project -lib/ +!lib/ !package*.json -node_modules -public -index.js +!public +!index.js !test -config.js \ No newline at end of file +!eslint.config.js +!config-default.js \ No newline at end of file diff --git a/InfoLogger/Dockerfile b/InfoLogger/Dockerfile index f26d9ef48..b3b4d5aed 100644 --- a/InfoLogger/Dockerfile +++ b/InfoLogger/Dockerfile @@ -1,6 +1,6 @@ # # ---- Base ---- -FROM node:22-alpine3.21 AS base +FROM node:22-alpine AS base RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app @@ -49,26 +49,6 @@ FROM developmentdependencies AS test # Run start script as specified in package.json CMD [ "npm", "run", "test" ] - - # # - # # ---- Test parallel for CI ---- - # FROM developmentdependencies AS test_parallel_ci - - # CMD [ "sh", "-c", "/opt/wait-for-it.sh -t 0 test_db:3306 -- npm run test:subset" ] - - # # - # # ---- Test parallel local ---- - # FROM developmentdependencies AS test_parallel_local - - # CMD [ "sh", "-c", "/opt/wait-for-it.sh -t 0 test_db:3306 -- npm run test:subset-local" ] - - # # - # # ---- Coverage ---- - # FROM developmentdependencies AS coverage - - # # Run start script as specified in package.json - # CMD [ "/opt/wait-for-it.sh", "-t", "0", "database:3306", "--", "npm", "run", "coverage" ] - # ---- Production Dependencies ---- FROM base AS productiondependencies @@ -79,12 +59,10 @@ COPY package*.json ./ # Installs modules from package-lock.json, this ensures reproducible build. RUN npm --silent ci --production -# Install nodemon (for development only) -RUN npm i nodemon - # Copy all files, filtered by .dockerignore, to the container with added dev dependancies. COPY . . RUN rm -r ./test +RUN rm eslint.config.js # ---- Simul ---- @@ -94,7 +72,7 @@ FROM developmentdependencies AS simul EXPOSE 6102 # Run simul script as specified in package.json -CMD [ "node", "run", "simul" ] +CMD [ "npm", "run", "simul" ] # ---- Production ---- FROM productiondependencies AS production diff --git a/InfoLogger/README.md b/InfoLogger/README.md index b0ded6f93..6f58df277 100644 --- a/InfoLogger/README.md +++ b/InfoLogger/README.md @@ -92,10 +92,13 @@ mysql: { ## Docker development -docker compose -f docker-compose.yaml -f docker-compose.dev.yml down -v -docker compose -f docker-compose.yaml -f docker-compose.dev.yml up -d --build +The development, test and simul docker image will take up about 1.35 GB, About 50% of that is due to puppeteers chromium requirement, the same reason why Electron apps are so big. +Should you want to run the ILG with the status quo (your changes applied) you can use the following command: +npm run docker:dev (will run the ILG with Nodemon, a dummy database, the ILG simulator and phpMyAdmin). +npm run docker:test (will run the ILG npm run test command inside a Docker container, use "docker logs --follow infologger-application-1" to see the results live). +npm run docker:simul (will run an ILG simulator with the port open so that whenever you run the ILG it will have live mode available). +npm run docker:cleanup (REMOVES all containers created with the above commands and their data). -process.env.mariadb_host ?? '127.0.0.1' ## Dummy InfoLogger test server InfoLoggerServer can be simulated by running `npm run simul`. The dummy server binds `localhost:6102` endpoint. diff --git a/InfoLogger/docker-compose.dev.yml b/InfoLogger/docker-compose.dev.yaml similarity index 87% rename from InfoLogger/docker-compose.dev.yml rename to InfoLogger/docker-compose.dev.yaml index 27b044290..75e4eac6f 100644 --- a/InfoLogger/docker-compose.dev.yml +++ b/InfoLogger/docker-compose.dev.yaml @@ -35,14 +35,15 @@ services: read_only: true source: ./index.js target: /home/node/app/index.js - database: + image: mariadb:latest + restart: unless-stopped environment: MARIADB_ROOT_PASSWORD: root - MARIADB_DATABASE: - ports: - - "3306:3306" - volumes: + # uncomment the code below should you want to access the DB using a external program. + # ports: + # - "3306:3306" + volumes: - type: bind read_only: true source: ./INFOLOGGER.sql @@ -63,15 +64,12 @@ services: - 9090:80 environment: - PMA_HOST=database - simulator: + simulator: build: target: simul environment: NODE_ENV: development - ports: - - "6102:6102" - command: "npm run simul" restart: "unless-stopped" - + volumes: - database-data: \ No newline at end of file + database-data: diff --git a/InfoLogger/docker-compose.simul.yaml b/InfoLogger/docker-compose.simul.yaml new file mode 100644 index 000000000..b70a60253 --- /dev/null +++ b/InfoLogger/docker-compose.simul.yaml @@ -0,0 +1,9 @@ +services: + simulator: + build: + target: simul + environment: + NODE_ENV: development + ports: + - "6102:6102" + restart: "unless-stopped" \ No newline at end of file diff --git a/InfoLogger/docker-compose.test.yaml b/InfoLogger/docker-compose.test.yaml new file mode 100644 index 000000000..f480ffada --- /dev/null +++ b/InfoLogger/docker-compose.test.yaml @@ -0,0 +1,28 @@ +services: + application: + build: + target: test + environment: + NODE_ENV: development + restart: "no" + volumes: + - type: bind + read_only: true + source: ./lib + target: /home/node/app/lib + - type: bind + read_only: true + source: ./public + target: /home/node/app/public + - type: bind + read_only: false + source: ./test + target: /home/node/app/test + - type: bind + read_only: true + source: ./config.js + target: /home/node/app/config.js + - type: bind + read_only: true + source: ./index.js + target: /home/node/app/index.js diff --git a/InfoLogger/docker-compose.yaml b/InfoLogger/docker-compose.yaml index 4ae570012..69796f5ef 100644 --- a/InfoLogger/docker-compose.yaml +++ b/InfoLogger/docker-compose.yaml @@ -3,9 +3,4 @@ services: build: context: . dockerfile: Dockerfile - target: production - database: - image: mariadb:latest - restart: unless-stopped - environment: - MARIADB_ROOT_PASSWORD: root + target: production \ No newline at end of file diff --git a/InfoLogger/package.json b/InfoLogger/package.json index 5e4907e0e..5cf8231d9 100644 --- a/InfoLogger/package.json +++ b/InfoLogger/package.json @@ -21,6 +21,10 @@ "start": "node index.js", "test": "npm run eslint && npm run mocha", "dev": "nodemon --inspect --watch index.js --watch lib --watch config.js index.js", + "docker:dev": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml up -d --build", + "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up -d --build", + "docker:simul": "docker compose -f docker-compose.simul.yaml up -d --build", + "docker:cleanup": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml down -v && docker compose -f docker-compose.yaml -f docker-compose.test.yaml down -v && docker compose -f docker-compose.simul.yaml down -v", "simul": "node test/live-simulator/runInfoLoggerServer.js", "eslint": "./node_modules/.bin/eslint --config eslint.config.js lib/ public/", "mocha": "mocha --exit $(find test -name 'mocha-*.js')", From c7ab1da941178b1b153dd95110b3fe92eedec1e4 Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Mon, 6 Jan 2025 17:46:26 +0100 Subject: [PATCH 05/10] Docs updated --- InfoLogger/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/InfoLogger/README.md b/InfoLogger/README.md index 6f58df277..12f6da6f5 100644 --- a/InfoLogger/README.md +++ b/InfoLogger/README.md @@ -93,11 +93,13 @@ mysql: { ## Docker development The development, test and simul docker image will take up about 1.35 GB, About 50% of that is due to puppeteers chromium requirement, the same reason why Electron apps are so big. -Should you want to run the ILG with the status quo (your changes applied) you can use the following command: -npm run docker:dev (will run the ILG with Nodemon, a dummy database, the ILG simulator and phpMyAdmin). -npm run docker:test (will run the ILG npm run test command inside a Docker container, use "docker logs --follow infologger-application-1" to see the results live). -npm run docker:simul (will run an ILG simulator with the port open so that whenever you run the ILG it will have live mode available). -npm run docker:cleanup (REMOVES all containers created with the above commands and their data). + +Should you want to run the ILG with the status quo (your changes applied) you can use the following commands: + +1. npm run docker:dev (will run the ILG with Nodemon, a dummy database, the ILG simulator and phpMyAdmin). +2. npm run docker:test (will run the ILG npm run test command inside a Docker container, use "docker logs --follow infologger-application-1" to see the results live). +3. npm run docker:simul (will run an ILG simulator with the port open so that whenever you run the ILG it will have live mode available). +4. npm run docker:cleanup (REMOVES all containers created with the above commands and their data). ## Dummy InfoLogger test server From 01bec0135e1f9fa96e9b4e278cf8b2e8dc3e4872 Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Mon, 6 Jan 2025 17:51:28 +0100 Subject: [PATCH 06/10] npm docker:test now prints results --- InfoLogger/README.md | 2 +- InfoLogger/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/InfoLogger/README.md b/InfoLogger/README.md index 12f6da6f5..f2b875d8d 100644 --- a/InfoLogger/README.md +++ b/InfoLogger/README.md @@ -97,7 +97,7 @@ The development, test and simul docker image will take up about 1.35 GB, About 5 Should you want to run the ILG with the status quo (your changes applied) you can use the following commands: 1. npm run docker:dev (will run the ILG with Nodemon, a dummy database, the ILG simulator and phpMyAdmin). -2. npm run docker:test (will run the ILG npm run test command inside a Docker container, use "docker logs --follow infologger-application-1" to see the results live). +2. npm run docker:test (will run the ILG npm run test command inside a Docker container and print the live log output). 3. npm run docker:simul (will run an ILG simulator with the port open so that whenever you run the ILG it will have live mode available). 4. npm run docker:cleanup (REMOVES all containers created with the above commands and their data). diff --git a/InfoLogger/package.json b/InfoLogger/package.json index 5cf8231d9..cb7de5493 100644 --- a/InfoLogger/package.json +++ b/InfoLogger/package.json @@ -22,7 +22,7 @@ "test": "npm run eslint && npm run mocha", "dev": "nodemon --inspect --watch index.js --watch lib --watch config.js index.js", "docker:dev": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml up -d --build", - "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up -d --build", + "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up -d --build && docker logs --follow infologger-application-1", "docker:simul": "docker compose -f docker-compose.simul.yaml up -d --build", "docker:cleanup": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml down -v && docker compose -f docker-compose.yaml -f docker-compose.test.yaml down -v && docker compose -f docker-compose.simul.yaml down -v", "simul": "node test/live-simulator/runInfoLoggerServer.js", From a78d87c760d40e7cc8c6ca29addd24b0428a1da4 Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Mon, 6 Jan 2025 17:58:12 +0100 Subject: [PATCH 07/10] documentation fixes --- InfoLogger/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InfoLogger/README.md b/InfoLogger/README.md index f2b875d8d..c38ddedc9 100644 --- a/InfoLogger/README.md +++ b/InfoLogger/README.md @@ -92,11 +92,11 @@ mysql: { ## Docker development -The development, test and simul docker image will take up about 1.35 GB, About 50% of that is due to puppeteers chromium requirement, the same reason why Electron apps are so big. +The development, test and simul docker image will take up about 1.35 GB, about 50% of that is due to puppeteer's chromium requirement, the same reason why Electron apps are so big. Should you want to run the ILG with the status quo (your changes applied) you can use the following commands: -1. npm run docker:dev (will run the ILG with Nodemon, a dummy database, the ILG simulator and phpMyAdmin). +1. npm run docker:dev (will run the ILG with Nodemon, a development database, the ILG simulator and phpMyAdmin). 2. npm run docker:test (will run the ILG npm run test command inside a Docker container and print the live log output). 3. npm run docker:simul (will run an ILG simulator with the port open so that whenever you run the ILG it will have live mode available). 4. npm run docker:cleanup (REMOVES all containers created with the above commands and their data). From 4acf9a58ef41e45f7a57884a482fa52857e9e58a Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Tue, 7 Jan 2025 09:37:18 +0100 Subject: [PATCH 08/10] disable Docker Deamon mode on dev and test config --- InfoLogger/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InfoLogger/package.json b/InfoLogger/package.json index cb7de5493..d6c3927d5 100644 --- a/InfoLogger/package.json +++ b/InfoLogger/package.json @@ -21,8 +21,8 @@ "start": "node index.js", "test": "npm run eslint && npm run mocha", "dev": "nodemon --inspect --watch index.js --watch lib --watch config.js index.js", - "docker:dev": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml up -d --build", - "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up -d --build && docker logs --follow infologger-application-1", + "docker:dev": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml up --build", + "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up --build", "docker:simul": "docker compose -f docker-compose.simul.yaml up -d --build", "docker:cleanup": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml down -v && docker compose -f docker-compose.yaml -f docker-compose.test.yaml down -v && docker compose -f docker-compose.simul.yaml down -v", "simul": "node test/live-simulator/runInfoLoggerServer.js", From 2b4fe6cab6dec97a6e5c9278eafd701dc761f3d2 Mon Sep 17 00:00:00 2001 From: Jasper Houweling Date: Tue, 7 Jan 2025 09:48:12 +0100 Subject: [PATCH 09/10] version pinning --- InfoLogger/docker-compose.dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InfoLogger/docker-compose.dev.yaml b/InfoLogger/docker-compose.dev.yaml index 75e4eac6f..bb59e1471 100644 --- a/InfoLogger/docker-compose.dev.yaml +++ b/InfoLogger/docker-compose.dev.yaml @@ -36,7 +36,7 @@ services: source: ./index.js target: /home/node/app/index.js database: - image: mariadb:latest + image: mariadb:11.6.2 restart: unless-stopped environment: MARIADB_ROOT_PASSWORD: root @@ -54,7 +54,7 @@ services: timeout: 20s retries: 20 phpmyadmin: - image: phpmyadmin + image: phpmyadmin:5.2.1 restart: unless-stopped depends_on: database: From aea439d18f05a396f6ebe5a5f291d8008471d17f Mon Sep 17 00:00:00 2001 From: Isaac Hill <71404865+isaachilly@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:44:38 +0200 Subject: [PATCH 10/10] Initial refactoring & streamlining of the dockerisation of ILG --- InfoLogger/Dockerfile | 6 ++-- InfoLogger/README.md | 39 +++++++++++++++++++++ InfoLogger/docker-compose.dev.yaml | 47 +++++++++++--------------- InfoLogger/{ => docker}/INFOLOGGER.sql | 0 InfoLogger/eslint.config.js | 8 +++++ InfoLogger/package.json | 27 ++++++++------- 6 files changed, 84 insertions(+), 43 deletions(-) rename InfoLogger/{ => docker}/INFOLOGGER.sql (100%) diff --git a/InfoLogger/Dockerfile b/InfoLogger/Dockerfile index b3b4d5aed..886de27c5 100644 --- a/InfoLogger/Dockerfile +++ b/InfoLogger/Dockerfile @@ -41,9 +41,9 @@ FROM developmentdependencies AS development EXPOSE 8080 # Run start script as specified in package.json -CMD [ "npm", "run", "dev" ] +CMD [ "npm", "run", "start:dev" ] - # # +# # ---- Test ---- FROM developmentdependencies AS test @@ -81,4 +81,4 @@ FROM productiondependencies AS production EXPOSE 8080 # Run start script as specified in package.json -CMD [ "node", "run", "start" ] \ No newline at end of file +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/InfoLogger/README.md b/InfoLogger/README.md index 6cf8fa5a3..57af9d82c 100644 --- a/InfoLogger/README.md +++ b/InfoLogger/README.md @@ -11,6 +11,7 @@ - [Interface User Guide](#interface-user-guide) - [Requirements](#requirements) - [Project Layout](#project-layout) + - [Scripts](#scripts) - [Backend](#backend) - [Frontend](#frontend) - [Local Development](#local-development) @@ -24,6 +25,8 @@ - [infologger.yml](#infologgeryml) - [release.yml](#releaseyml) - [system-configuration pipeline](#system-configuration-pipeline) + - [Docker development](#docker-development) + - [Development database installation](#development-database-installation) - [InfoLogger Insights](#infologger-insights) ## Introduction @@ -61,6 +64,17 @@ Screenshot of the current interface, running locally against a fake InfoLoggerSe ## Project Layout ILG is a Node.js API Gateway and Single-Page Application built on top of the `@aliceo2/web-ui` framework. It serves as a unified interface to two separate operational backends: a **MariaDB database** for historical queries and an **InfoLoggerServer TCP endpoint** for live log streaming. +## Scripts +| Script | Description | +| --- | --- | +| `npm start` | Run the app (`node index.js`). | +| `npm run lint` | Lint the project with ESLint. | +| `npm run lint:fix` | Lint and automatically fix fixable problems. | +| `npm test` | Run the linter, then the Mocha test suite. | +| `npm run coverage` | Run the test suite under `nyc` and generate the coverage report. | +| `npm run coverage:report` | Generate HTML + JSON coverage reports under `coverage/` from the last run. | + +Docker commands (`docker:run`, `docker:test`, `docker:simul`, `docker:cleanup`) are documented under [Docker development](#docker-development). ## Backend @@ -195,6 +209,31 @@ Should you want to run the ILG with the status quo (your changes applied) you ca 3. npm run docker:simul (will run an ILG simulator with the port open so that whenever you run the ILG it will have live mode available). 4. npm run docker:cleanup (REMOVES all containers created with the above commands and their data). +### Development database installation +In order to run queries in the InfoLogger a MariaDB server is required. To run a local MariaDB server that can easily be updated/wiped/configured you will need Docker installed. +1. Follow the instructions specific for you platform: [docker desktop install](https://www.docker.com/products/docker-desktop/) +2. Create a `compose.yaml` file somewhere on your pc with the following content: +``` +services: + mariadb: + image: mariadb + restart: unless-stopped + ports: + - 3306:3306 + environment: + MARIADB_ROOT_PASSWORD: root + # (this is just an example, not intended to be a production configuration) + phpmyadmin: + image: phpmyadmin + restart: unless-stopped + ports: + - 9090:80 + environment: + - PMA_HOST=mariadb +``` +This will get you a MariaDB server with phpmyadmin. +Should you ever feel the need to test a specific version of MariaDB then change the image to `mariadb:11.5` where 11.5 is the version. By default the compose.yaml will get you the latest MariaDB image. + ## InfoLogger Insights - [InfoLogger message protocol](docs/il-protocol.md) diff --git a/InfoLogger/docker-compose.dev.yaml b/InfoLogger/docker-compose.dev.yaml index bb59e1471..2c86e8987 100644 --- a/InfoLogger/docker-compose.dev.yaml +++ b/InfoLogger/docker-compose.dev.yaml @@ -2,18 +2,22 @@ services: application: build: target: development + + restart: "no" + depends_on: database: condition: service_healthy restart: true + environment: NODE_ENV: development mariadb_host: database infologger_host: simulator + ports: - "8080:8080" - command: "npm run dev" - restart: "no" + volumes: - type: bind read_only: true @@ -23,10 +27,6 @@ services: read_only: true source: ./public target: /home/node/app/public - - type: bind - read_only: true - source: ./test - target: /home/node/app/test - type: bind read_only: true source: ./config.js @@ -38,38 +38,31 @@ services: database: image: mariadb:11.6.2 restart: unless-stopped + environment: MARIADB_ROOT_PASSWORD: root - # uncomment the code below should you want to access the DB using a external program. - # ports: - # - "3306:3306" + + # should you want to access the DB using a external program + ports: + - "3306:3306" + + # tmpfs keeps the data dir in RAM, so it starts empty on every run and + # MariaDB re-applies INFOLOGGER.sql each time (throwaway dev database). volumes: + - type: tmpfs + target: /var/lib/mysql - type: bind read_only: true - source: ./INFOLOGGER.sql + source: ./docker/INFOLOGGER.sql target: /docker-entrypoint-initdb.d/INFOLOGGER.sql + healthcheck: test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] - interval: 1s + interval: 5s timeout: 20s retries: 20 - phpmyadmin: - image: phpmyadmin:5.2.1 - restart: unless-stopped - depends_on: - database: - condition: service_healthy - restart: true - ports: - - 9090:80 - environment: - - PMA_HOST=database + simulator: build: target: simul - environment: - NODE_ENV: development restart: "unless-stopped" - -volumes: - database-data: diff --git a/InfoLogger/INFOLOGGER.sql b/InfoLogger/docker/INFOLOGGER.sql similarity index 100% rename from InfoLogger/INFOLOGGER.sql rename to InfoLogger/docker/INFOLOGGER.sql diff --git a/InfoLogger/eslint.config.js b/InfoLogger/eslint.config.js index 7d58ac2f6..902af3269 100644 --- a/InfoLogger/eslint.config.js +++ b/InfoLogger/eslint.config.js @@ -18,6 +18,14 @@ const jsdoc = require('eslint-plugin-jsdoc'); const stylisticJs = require('@stylistic/eslint-plugin'); module.exports = [ + { + ignores: [ + 'test/', + 'node_modules/', + 'tmp/', + 'nyc_output/', + ], + }, jsdoc.configs['flat/recommended'], pluginJs.configs.recommended, { diff --git a/InfoLogger/package.json b/InfoLogger/package.json index 5953ae1a1..5c24ec9cb 100644 --- a/InfoLogger/package.json +++ b/InfoLogger/package.json @@ -18,19 +18,20 @@ }, "homepage": "https://alice-o2-project.web.cern.ch/", "scripts": { - "start": "node index.js", - "test": "npm run eslint && npm run mocha", - "dev": "nodemon --inspect --watch index.js --watch lib --watch config.js index.js", - "docker:dev": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml up --build", - "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up --build", - "docker:simul": "docker compose -f docker-compose.simul.yaml up -d --build", - "docker:cleanup": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml down -v && docker compose -f docker-compose.yaml -f docker-compose.test.yaml down -v && docker compose -f docker-compose.simul.yaml down -v", - "simul": "node test/live-simulator/runInfoLoggerServer.js", - "eslint": "./node_modules/.bin/eslint --config eslint.config.js lib/ public/", - "mocha": "mocha --exit $(find test -name 'mocha-*.js')", - "coverage": "npm run eslint && nyc npm run mocha", - "coverage-local": "nyc --reporter=lcov npm run mocha" - }, + "lint": "eslint --config eslint.config.js", + "lint:fix": "npm run lint -- --fix", + "test": "npm run lint && npm run mocha", + "mocha": "mocha --exit $(find test -name 'mocha-*.js')", + "coverage": "nyc npm test && npm run coverage:report", + "coverage:report": "nyc report --reporter=html --reporter=json", + "start": "node index.js", + "start:dev": "nodemon --inspect --watch index.js --watch lib --watch config.js index.js", + "simul": "node test/live-simulator/runInfoLoggerServer.js", + "docker:dev": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml up --build", + "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up --build", + "docker:simul": "docker compose -f docker-compose.simul.yaml up -d --build", + "docker:cleanup": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml -f docker-compose.test.yaml -f docker-compose.simul.yaml down -v --remove-orphans" +}, "main": "index.js", "dependencies": { "@aliceo2/web-ui": "2.11.0",