Continuing Aurash's work on MSQT CI workflow

.github/actions/setup-dev-nightly/action.yml

@@ -10,7 +10,7 @@ runs:
         mkdir nightly
         cd nightly
         source /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh
-        setup_dbt latest_v5
+        setup_dbt latest
         dbt-create -n last_fddaq
         cd NFD_DEV*
         source env.sh

.github/workflows/check_dirs_init_file.yml

@@ -21,6 +21,7 @@ jobs:
             -path "src/tests*" -o \
             -path "src/docs*" -o \
             -path "src/config*" -o \
+            -path "src/drunc/integtest*" -o \
             -path "src/drunc/data*" \) -prune -o \
             -type d -exec sh -c '[ ! -e "$1/__init__.py" ] && echo "$1"' _ {} \;)
 

.github/workflows/check_docs.yml

@@ -2,22 +2,26 @@ name: Check Links In Docs
 
 on:
   pull_request:
-    paths:
-      - 'docs/**'
-    workflow_dispatch: 
-    workflow_call:
-    schedule:
-      - cron: "0 0 * * 6" # midnight every Saturday
+  workflow_dispatch:
+  workflow_call:
+  schedule:
+    - cron: "0 0 * * 6" # Runs every Saturday at midnight
 
 jobs:
   check-links:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: Run Lychee link checker
         uses: lycheeverse/lychee-action@v2
         with:
-          args: docs/
-
+          args: >
+            --exclude 'https://twiki.cern.ch/.*'
+            --exclude 'https://indico.*'
+            --exclude 'https://wiki.dunescience.org/wiki/.*'
+            --exclude 'https://github.com/DUNE-DAQ/drunc/wiki/Archive:.*'
+            docs/

.github/workflows/run_msqt.yml

@@ -36,23 +36,32 @@ jobs:
         run: |
           cd nightly/NFD_DEV* || true
           source env.sh
-          python -m pip install --upgrade pip
           cd sourcecode
+          echo "Checking part of the git clone resolution path: ${{ github.event.pull_request.head.repo.owner.login }}"
           git clone https://github.com/${{ github.event.pull_request.head.repo.owner.login }}/druncschema.git -b ${{ github.head_ref }} || git clone https://github.com/DUNE-DAQ/druncschema.git
-          git clone https://github.com/${{ github.event.pull_request.head.repo.owner.login }}/drunc.git -b ${{ github.head_ref }} || git clone https://github.com/DUNE-DAQ/drunc.git
           cd druncschema
           pip install .[test]
-          cd -
+          cd ../../pythoncode
+          git clone https://github.com/${{ github.event.pull_request.head.repo.owner.login }}/drunc.git -b ${{ github.head_ref }} || git clone https://github.com/DUNE-DAQ/drunc.git
           cd drunc
-          pip install -e .[dev]
+          pip install .[test]
           echo "DRUNC_DIR=$(pwd)" >> $GITHUB_ENV
 
       - name: Test with MSQT
         if: success()
         run: |
           cd nightly/NFD_DEV* || true
           source env.sh
-          daqsystemtest_integtest_bundle.sh --stop-on-failure --pytest-options "--tb=long" -k small_footprint_quick_test.py 2>&1
+          echo "Check the permissions of the path used for testing"
+          ls -lahrt .
+          echo "Check the repo structure before running tests"
+          ls -lahrt sourcecode
+          ls -lahrt pythoncode
+          echo "Run the tests as the current user: $(whoami)"
+          echo 'Test command: dunedaq_integtest_bundle.sh --stop-on-failure --pytest-options "--skip-resource-checks" -k small_footprint_quick_test.py --verbosity 5 --tmpdir . 2>&1'
+          dunedaq_integtest_bundle.sh --stop-on-failure --pytest-options '--skip-resource-checks' -k small_footprint_quick_test.py --verbosity 5 --tmpdir . 2>&1
+          echo "ECHOING THE INTEGRATION TEST BUNDLE LOG FILE"
+          cat pytest-of-*/dunedaq_integtest_bundle*.log
 
       - name: Print test log
         if: always()

docs/Messaging-format.md

@@ -79,14 +79,12 @@ message Description {
   string name = 2;
   optional string session = 3;
   repeated CommandDescription commands = 4;
-  optional google.protobuf.Any broadcast = 5;
 }
 ```
 * `type` can be `process_manager` or `controller` right now it is a string.
 * `name` is the name of the server.
 * `session` is the (optional) session name.
 * `commands` is a vector of acceptable commands to send to the endpoint.
-* `broadcast` is a description of the broadcast service.
 
 `CommandDescription` is used to describe all the commands, it has the following format:
 ```
@@ -102,58 +100,3 @@ message CommandDescription {
 * `help` is a string providing help
 * `return_type` is the format of the `data` field that should be expected inside the `Response` if the command execution was successful.
 
-For broadcasting, there is right now only one format that the `describe` command can fill, with a description of the Kafka service that is used to broadcast the log messages:
-```
-message KafkaBroadcastHandlerConfiguration{
-  string kafka_address = 1;
-  string topic = 2;
-}
-```
-* `kafka_address` is a bootstrap server
-* `topic` is the Kafka topic on which this service is logging.
-
-More formats of broadcasting may be added in the future (potentially using [ERS's python binding](https://github.com/DUNE-DAQ/erskafka/tree/develop/python/erskafka)).
-
-# Broadcasting
-As mentioned earlier, the only working solution for broadcasting is Kafka. All messages feed to Kafka by drunc have the [form](https://github.com/DUNE-DAQ/druncschema/blob/develop/schema/druncschema/broadcast.proto#L51):
-```
-message BroadcastMessage{
-  Emitter emitter = 1;
-  BroadcastType type = 2;
-  google.protobuf.Any data = 3;
-}
-```
-Where:
-```
-message Emitter {
-  string process = 1;
-  string session = 2;
-}
-```
-and
-```
-enum BroadcastType {
-  ACK                             = 0;
-  RECEIVER_REMOVED                = 1;
-  RECEIVER_ADDED                  = 2;
-  SERVER_READY                    = 3;
-  SERVER_SHUTDOWN                 = 4;
-  TEXT_MESSAGE                    = 15;
-  COMMAND_EXECUTION_START         = 5;
-  COMMAND_RECEIVED                = 16;
-  COMMAND_EXECUTION_SUCCESS       = 6;
-  EXCEPTION_RAISED                = 7;
-  UNHANDLED_EXCEPTION_RAISED      = 8;
-  STATUS_UPDATE                   = 9;
-  SUBPROCESS_STATUS_UPDATE        = 10;
-  DEBUG                           = 11;
-  CHILD_COMMAND_EXECUTION_START   = 12;
-  CHILD_COMMAND_EXECUTION_SUCCESS = 13;
-  CHILD_COMMAND_EXECUTION_FAILED  = 14;
-  FSM_STATUS_UPDATE               = 17;
-}
-```
-
-Note that for now, the `data` field is only filled with [PlainText](https://github.com/DUNE-DAQ/druncschema/blob/develop/schema/druncschema/generic.proto#L7C1-L9C2) messages.
-
-In the future, this may change.

docs/Process-manager-interface.md

@@ -193,7 +193,7 @@ message ExceptionNotification {
 Each RPC call is described here.
 
 ### `describe`
-Returns metadata about the process manager, including available commands, session info, and broadcast description.
+Returns metadata about the process manager, including available commands, and session info.
 * input: `Request`
 * output: [Description](https://dune-daq-sw.readthedocs.io/en/latest/packages/drunc/Messaging-format)
 

docs/Process-manager.md

@@ -7,7 +7,7 @@ For a standalone `process_manager` you will need two shells - one shell to run t
 To boot a `process_manager`, you will need to choose the most appropriate configuration that applies to the use case. The configurations that are packaged with `drunc` are defined in `drunc/src/data/process_manager/`, which are
 * `ssh-standalone.json`: `ssh` based implementation without a `kafka` feed. Uses remote shell client processes to manage ssh connections.
 * `ssh-standalone-paramiko-client.json`: `ssh` based implementation without a `kafka` feed. Uses paramiko library to manage ssh connections from the python process. Currently NOT maintained but may be revisited in future - use `ssh-standalone` only for now.
-* `ssh-pocket-kafka.json`: `ssh` based implementation with `pocket`'s `kafka` for message broadcasting.
+* `ssh-pocket-kafka.json`: `ssh` based implementation with `pocket`'s `kafka`.
 * `ssh-CERN-kafka.json`: `ssh` based implementation with `kafka` service running at ENH1.
 * `ssh-CERN-kafka-OpMon.json`: `ssh` based implementation with `kafka` service running at ENH1, and with Opmon.
 * `k8s.json`: `kubernetes` implementation (not recommended nor working, so don't use this unless you are an working on getting it to work).
@@ -26,8 +26,6 @@ To start the ssh version without kafka:
 drunc-process-manager ssh-standalone
 Using 'file://src/drunc/data/process_manager/ssh-standalone.json' as the ProcessManager configuration
 Starting 'SSHProcessManager'
-[12:43:26] INFO     "BroadcastSenderConfHandler": None                                                                                                                                                  configuration.py:25
-           INFO     "Controller": DummyAuthoriser ready                                                                                                                                              dummy_authoriser.py:13
 ProcessManager was started on np04-srv-019:10054
 ```
 Once this is done, you will not be able to send commands to the process from the current shell with the `process_manager` acting in the foreground. To interact with a standalone instance of `process_manager` you will need to connect to it (see below).

docs/Unified-shell-reference.md

@@ -414,7 +414,6 @@ logs -n root-controller --how-far 5
            INFO     "Controller": 'df-controller@localhost:5600' (type ChildNodeType.gRPC)                                           controller.py:123
            INFO     "Controller": 'trg-controller@localhost:5700' (type ChildNodeType.gRPC)                                          controller.py:123
            INFO     "Controller": 'hsi-controller@localhost:5800' (type ChildNodeType.gRPC)                                          controller.py:123
-           INFO     "Broadcast": ready                                                                                          broadcast_sender.py:65
 root-controller was started on localhost:3333
 ───────────────────────────────────────────────────────────────────────────────────────────── End ──────────────────────────────────────────────────────────────────────────────────────────────
 ```

scripts/drunc_integtest_bundle.sh

@@ -5,7 +5,7 @@
 # Based entirely of the implementation of daqsystemtest_integtest_bundle.sh
 # Original author: KAB, 10-Oct-2023
 
-integtest_list=( "process_manager_test.py" )
+integtest_list=( "process_manager_test.py" "controller_test.py" )
 let last_test_index=${#integtest_list[@]}-1
 
 usage() {

src/drunc/authoriser/configuration.py

@@ -2,4 +2,7 @@
 
 
 class DummyAuthoriserConfHandler(ConfHandler):
-    pass
+    """Handler for dummy authoriser configuration."""
+
+    def populate_from_dict(self, data: dict[str, object]) -> None:
+        pass

src/drunc/authoriser/dummy_authoriser.py

@@ -5,22 +5,21 @@
 from drunc.utils.utils import get_logger
 
 
-# TODO: Should be communicating over network
-# The Rolls Royce of the authoriser systems
 class DummyAuthoriser:
     def __init__(
-        self,
-        system: SystemType,
-        configuration_handler: DummyAuthoriserConfHandler = None,
+        self, configuration: DummyAuthoriserConfHandler, system: SystemType.ValueType
     ):
         self.log = get_logger("utils.authorizer")
         self.log.debug("DummyAuthoriser ready")
-        self.configuration = configuration_handler
-        self.command_actions = {}  # Dict[str, ActionType]
+        self.configuration = configuration
         self.system = system
 
     def is_authorised(
-        self, token: Token, action: ActionType, system: SystemType, cmd_name: str = ""
+        self,
+        token: Token,
+        action: ActionType.ValueType,
+        system: SystemType.ValueType,
+        cmd_name: str,
     ) -> bool:
         self.log.debug(
             f"Authorising {token.user_name} to {ActionType.Name(action)} ({cmd_name}) on {SystemType.Name(system)}"
@@ -30,11 +29,3 @@ def is_authorised(
     def authorised_actions(self, token: Token) -> list[str]:
         self.log.info(f"Grabbing authorisations for {token.token}")
         return []
-
-
-def main():
-    DummyAuthoriser()
-
-
-if __name__ == "__main__":
-    main()