diff --git a/InfoLogger/.dockerignore b/InfoLogger/.dockerignore new file mode 100644 index 000000000..b1a793ffd --- /dev/null +++ b/InfoLogger/.dockerignore @@ -0,0 +1,11 @@ +# Default ignore everything +* + +# Project +!lib/ +!package*.json +!public +!index.js +!test +!eslint.config.js +!config-default.js \ No newline at end of file diff --git a/InfoLogger/Dockerfile b/InfoLogger/Dockerfile new file mode 100644 index 000000000..886de27c5 --- /dev/null +++ b/InfoLogger/Dockerfile @@ -0,0 +1,84 @@ +# +# ---- Base ---- +FROM node:22-alpine AS base + +RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app + +# Create app directory +WORKDIR /home/node/app + +# +# ---- Development Dependencies ---- +FROM base AS developmentdependencies + +# Installs Git and packages required for Puppeteer +# https://pkgs.alpinelinux.org/packages +RUN apk add --no-cache \ + chromium \ + freetype \ + freetype-dev + +# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + +# Copy all files, except those ignored by .dockerignore, to the container +COPY . . + + + +# Installs modules from package-lock.json if there are changes, this ensures reproducible build +RUN npm --silent ci + +# Install nodemon (for development only) +RUN npm i nodemon + +# +# ---- Development ---- +FROM developmentdependencies AS development + +# Expose the port to the Docker instance (not the host!) +EXPOSE 8080 + +# Run start script as specified in package.json +CMD [ "npm", "run", "start:dev" ] + +# +# ---- Test ---- +FROM developmentdependencies AS test + +# Run start script as specified in package.json +CMD [ "npm", "run", "test" ] + +# ---- Production Dependencies ---- +FROM base AS productiondependencies + +# Copy package.json and package-lock.json to the container. +COPY package*.json ./ + +# Installs modules from package-lock.json, this ensures reproducible build. +RUN npm --silent ci --production + + # Copy all files, filtered by .dockerignore, to the container with added dev dependancies. +COPY . . +RUN rm -r ./test +RUN rm eslint.config.js + + +# ---- Simul ---- +FROM developmentdependencies AS simul + +# Expose simul server port. +EXPOSE 6102 + +# Run simul script as specified in package.json +CMD [ "npm", "run", "simul" ] + +# ---- Production ---- +FROM productiondependencies AS production + +# Expose the port to the Docker instance (not the host!) +EXPOSE 8080 + +# Run start script as specified in package.json +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/InfoLogger/README.md b/InfoLogger/README.md index 676e37a56..57af9d82c 100644 --- a/InfoLogger/README.md +++ b/InfoLogger/README.md @@ -11,6 +11,7 @@ - [Interface User Guide](#interface-user-guide) - [Requirements](#requirements) - [Project Layout](#project-layout) + - [Scripts](#scripts) - [Backend](#backend) - [Frontend](#frontend) - [Local Development](#local-development) @@ -24,6 +25,8 @@ - [infologger.yml](#infologgeryml) - [release.yml](#releaseyml) - [system-configuration pipeline](#system-configuration-pipeline) + - [Docker development](#docker-development) + - [Development database installation](#development-database-installation) - [InfoLogger Insights](#infologger-insights) ## Introduction @@ -61,6 +64,17 @@ Screenshot of the current interface, running locally against a fake InfoLoggerSe ## Project Layout ILG is a Node.js API Gateway and Single-Page Application built on top of the `@aliceo2/web-ui` framework. It serves as a unified interface to two separate operational backends: a **MariaDB database** for historical queries and an **InfoLoggerServer TCP endpoint** for live log streaming. +## Scripts +| Script | Description | +| --- | --- | +| `npm start` | Run the app (`node index.js`). | +| `npm run lint` | Lint the project with ESLint. | +| `npm run lint:fix` | Lint and automatically fix fixable problems. | +| `npm test` | Run the linter, then the Mocha test suite. | +| `npm run coverage` | Run the test suite under `nyc` and generate the coverage report. | +| `npm run coverage:report` | Generate HTML + JSON coverage reports under `coverage/` from the last run. | + +Docker commands (`docker:run`, `docker:test`, `docker:simul`, `docker:cleanup`) are documented under [Docker development](#docker-development). ## Backend @@ -184,6 +198,42 @@ Runs on every PR touching `InfoLogger/**`. Must pass to merge: Runs the integration suite against a pipeline instance. Runs separately from this repo's CI, so run the suite locally before merging and flag any required changes to the team at release time. +## Docker development + +The development, test and simul docker image will take up about 1.35 GB, about 50% of that is due to puppeteer's chromium requirement, the same reason why Electron apps are so big. + +Should you want to run the ILG with the status quo (your changes applied) you can use the following commands: + +1. npm run docker:dev (will run the ILG with Nodemon, a development database, the ILG simulator and phpMyAdmin). +2. npm run docker:test (will run the ILG npm run test command inside a Docker container and print the live log output). +3. npm run docker:simul (will run an ILG simulator with the port open so that whenever you run the ILG it will have live mode available). +4. npm run docker:cleanup (REMOVES all containers created with the above commands and their data). + +### Development database installation +In order to run queries in the InfoLogger a MariaDB server is required. To run a local MariaDB server that can easily be updated/wiped/configured you will need Docker installed. +1. Follow the instructions specific for you platform: [docker desktop install](https://www.docker.com/products/docker-desktop/) +2. Create a `compose.yaml` file somewhere on your pc with the following content: +``` +services: + mariadb: + image: mariadb + restart: unless-stopped + ports: + - 3306:3306 + environment: + MARIADB_ROOT_PASSWORD: root + # (this is just an example, not intended to be a production configuration) + phpmyadmin: + image: phpmyadmin + restart: unless-stopped + ports: + - 9090:80 + environment: + - PMA_HOST=mariadb +``` +This will get you a MariaDB server with phpmyadmin. +Should you ever feel the need to test a specific version of MariaDB then change the image to `mariadb:11.5` where 11.5 is the version. By default the compose.yaml will get you the latest MariaDB image. + ## InfoLogger Insights - [InfoLogger message protocol](docs/il-protocol.md) diff --git a/InfoLogger/config-default.js b/InfoLogger/config-default.js index dbf38c686..b02a218b1 100644 --- a/InfoLogger/config-default.js +++ b/InfoLogger/config-default.js @@ -28,7 +28,7 @@ module.exports = { // optional data source, comment object if not used // all options: https://github.com/mysqljs/mysql#connection-options mysql: { - host: '127.0.0.1', + host: process.env.mariadb_host ?? '127.0.0.1', user: 'root', password: 'root', database: 'INFOLOGGER', @@ -40,7 +40,7 @@ module.exports = { // optional data source, comment object if not used // all options: https://nodejs.org/api/net.html#net_socket_connect_options_connectlistener infoLoggerServer: { - host: 'localhost', + host: process.env.infologger_host ?? 'localhost', port: 6102, }, logging: { diff --git a/InfoLogger/docker-compose.dev.yaml b/InfoLogger/docker-compose.dev.yaml new file mode 100644 index 000000000..2c86e8987 --- /dev/null +++ b/InfoLogger/docker-compose.dev.yaml @@ -0,0 +1,68 @@ +services: + application: + build: + target: development + + restart: "no" + + depends_on: + database: + condition: service_healthy + restart: true + + environment: + NODE_ENV: development + mariadb_host: database + infologger_host: simulator + + ports: + - "8080:8080" + + volumes: + - type: bind + read_only: true + source: ./lib + target: /home/node/app/lib + - type: bind + read_only: true + source: ./public + target: /home/node/app/public + - type: bind + read_only: true + source: ./config.js + target: /home/node/app/config.js + - type: bind + read_only: true + source: ./index.js + target: /home/node/app/index.js + database: + image: mariadb:11.6.2 + restart: unless-stopped + + environment: + MARIADB_ROOT_PASSWORD: root + + # should you want to access the DB using a external program + ports: + - "3306:3306" + + # tmpfs keeps the data dir in RAM, so it starts empty on every run and + # MariaDB re-applies INFOLOGGER.sql each time (throwaway dev database). + volumes: + - type: tmpfs + target: /var/lib/mysql + - type: bind + read_only: true + source: ./docker/INFOLOGGER.sql + target: /docker-entrypoint-initdb.d/INFOLOGGER.sql + + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 20s + retries: 20 + + simulator: + build: + target: simul + restart: "unless-stopped" diff --git a/InfoLogger/docker-compose.simul.yaml b/InfoLogger/docker-compose.simul.yaml new file mode 100644 index 000000000..b70a60253 --- /dev/null +++ b/InfoLogger/docker-compose.simul.yaml @@ -0,0 +1,9 @@ +services: + simulator: + build: + target: simul + environment: + NODE_ENV: development + ports: + - "6102:6102" + restart: "unless-stopped" \ No newline at end of file diff --git a/InfoLogger/docker-compose.test.yaml b/InfoLogger/docker-compose.test.yaml new file mode 100644 index 000000000..f480ffada --- /dev/null +++ b/InfoLogger/docker-compose.test.yaml @@ -0,0 +1,28 @@ +services: + application: + build: + target: test + environment: + NODE_ENV: development + restart: "no" + volumes: + - type: bind + read_only: true + source: ./lib + target: /home/node/app/lib + - type: bind + read_only: true + source: ./public + target: /home/node/app/public + - type: bind + read_only: false + source: ./test + target: /home/node/app/test + - type: bind + read_only: true + source: ./config.js + target: /home/node/app/config.js + - type: bind + read_only: true + source: ./index.js + target: /home/node/app/index.js diff --git a/InfoLogger/docker-compose.yaml b/InfoLogger/docker-compose.yaml new file mode 100644 index 000000000..69796f5ef --- /dev/null +++ b/InfoLogger/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + application: + build: + context: . + dockerfile: Dockerfile + target: production \ No newline at end of file diff --git a/InfoLogger/docker/INFOLOGGER.sql b/InfoLogger/docker/INFOLOGGER.sql new file mode 100644 index 000000000..4b33964d9 --- /dev/null +++ b/InfoLogger/docker/INFOLOGGER.sql @@ -0,0 +1,87 @@ +-- phpMyAdmin SQL Dump +-- version 5.2.1 +-- https://www.phpmyadmin.net/ +-- +-- Host: mariadb +-- Generation Time: Dec 15, 2024 at 02:28 PM +-- Server version: 11.5.2-MariaDB-ubu2404 +-- PHP Version: 8.2.25 + +SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; +START TRANSACTION; +SET time_zone = "+00:00"; + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; + +-- +-- Database: `INFOLOGGER` +-- +CREATE DATABASE IF NOT EXISTS `INFOLOGGER` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_uca1400_ai_ci; +USE `INFOLOGGER`; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `messages` +-- + +CREATE TABLE `messages` ( + `severity` char(1) DEFAULT NULL, + `level` tinyint(3) UNSIGNED DEFAULT NULL, + `timestamp` double(16,6) DEFAULT NULL, + `hostname` varchar(32) DEFAULT NULL, + `rolename` varchar(32) DEFAULT NULL, + `pid` smallint(5) UNSIGNED DEFAULT NULL, + `username` varchar(32) DEFAULT NULL, + `system` varchar(32) DEFAULT NULL, + `facility` varchar(32) DEFAULT NULL, + `detector` varchar(32) DEFAULT NULL, + `partition` varchar(32) DEFAULT NULL, + `dest` varchar(32) DEFAULT NULL, + `run` int(10) UNSIGNED DEFAULT NULL, + `errcode` int(10) UNSIGNED DEFAULT NULL, + `errline` smallint(5) UNSIGNED DEFAULT NULL, + `errsource` varchar(32) DEFAULT NULL, + `message` text DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +PARTITION BY HASH (`timestamp` DIV 86400) +PARTITIONS 365; + +-- +-- Dumping data for table `messages` +-- + +INSERT INTO `messages` (`severity`, `level`, `timestamp`, `hostname`, `rolename`, `pid`, `username`, `system`, `facility`, `detector`, `partition`, `dest`, `run`, `errcode`, `errline`, `errsource`, `message`) VALUES +('I', 6, 1456134840.001504, 'aldaqecs01-v1', NULL, 2733, 'alicedaq', 'DAQ', 'LHCBeamInfo', NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'Mon Feb 22 10:54:00 2016 (1456134840): BeamMode: NO BEAM; BeamType: ; ParticleTypeB1: ; ParticleTypeB2: ; BetaStar: ; Eng: 0 GeV; FillN: -999; FillScheme: ; NbunchesInt: 0; NbunchesNotIntB1: 0; NbunchesNotIntB2: 0; InteracIst I(B1): 0; InteracTot I(B1): 0; InteracIst I(B2): 0; InteracTot I(B2): 0; NotInteracIst I(B1): 0; NotInteracTot I(B1): 0; NotInteracIst I(B2): 0; NotInteracTot I(B2): 0; PostMortemN: 1; LHCAdjust: STANDBY; LHCBeamDump: STANDBY; LHCInjection: STANDBY; DIPconnected Flag: 1 1 1 1 1 1 1 1 1 1 1 1 1 1'); + +-- +-- Indexes for dumped tables +-- + +-- +-- Indexes for table `messages` +-- +ALTER TABLE `messages` + ADD KEY `ix_severity` (`severity`), + ADD KEY `ix_level` (`level`), + ADD KEY `ix_timestamp` (`timestamp`), + ADD KEY `ix_hostname` (`hostname`(14)), + ADD KEY `ix_rolename` (`rolename`(20)), + ADD KEY `ix_system` (`system`(3)), + ADD KEY `ix_facility` (`facility`(20)), + ADD KEY `ix_detector` (`detector`(8)), + ADD KEY `ix_partition` (`partition`(10)), + ADD KEY `ix_dest` (`dest`(10)), + ADD KEY `ix_run` (`run`), + ADD KEY `ix_errcode` (`errcode`), + ADD KEY `ix_errline` (`errline`), + ADD KEY `ix_errsource` (`errsource`(20)); +COMMIT; + +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; diff --git a/InfoLogger/eslint.config.js b/InfoLogger/eslint.config.js index 7d58ac2f6..902af3269 100644 --- a/InfoLogger/eslint.config.js +++ b/InfoLogger/eslint.config.js @@ -18,6 +18,14 @@ const jsdoc = require('eslint-plugin-jsdoc'); const stylisticJs = require('@stylistic/eslint-plugin'); module.exports = [ + { + ignores: [ + 'test/', + 'node_modules/', + 'tmp/', + 'nyc_output/', + ], + }, jsdoc.configs['flat/recommended'], pluginJs.configs.recommended, { diff --git a/InfoLogger/package.json b/InfoLogger/package.json index f38a3b233..5c24ec9cb 100644 --- a/InfoLogger/package.json +++ b/InfoLogger/package.json @@ -18,15 +18,20 @@ }, "homepage": "https://alice-o2-project.web.cern.ch/", "scripts": { - "start": "node index.js", - "test": "npm run eslint && npm run mocha", - "dev": "nodemon --inspect --watch index.js --watch lib --watch config.js index.js", - "simul": "node test/live-simulator/runInfoLoggerServer.js", - "eslint": "./node_modules/.bin/eslint --config eslint.config.js lib/ public/", - "mocha": "mocha --exit $(find test -name 'mocha-*.js')", - "coverage": "npm run eslint && nyc npm run mocha", - "coverage-local": "nyc --reporter=lcov npm run mocha" - }, + "lint": "eslint --config eslint.config.js", + "lint:fix": "npm run lint -- --fix", + "test": "npm run lint && npm run mocha", + "mocha": "mocha --exit $(find test -name 'mocha-*.js')", + "coverage": "nyc npm test && npm run coverage:report", + "coverage:report": "nyc report --reporter=html --reporter=json", + "start": "node index.js", + "start:dev": "nodemon --inspect --watch index.js --watch lib --watch config.js index.js", + "simul": "node test/live-simulator/runInfoLoggerServer.js", + "docker:dev": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml up --build", + "docker:test": "docker compose -f docker-compose.yaml -f docker-compose.test.yaml up --build", + "docker:simul": "docker compose -f docker-compose.simul.yaml up -d --build", + "docker:cleanup": "docker compose -f docker-compose.yaml -f docker-compose.dev.yaml -f docker-compose.test.yaml -f docker-compose.simul.yaml down -v --remove-orphans" +}, "main": "index.js", "dependencies": { "@aliceo2/web-ui": "2.11.0",