migrid-ux based peers ui and all supporting logic

.github/workflows/ci.yml

@@ -93,10 +93,10 @@ jobs:
           dnf install -y python3 python3-pip make
       - name: Setup environment
         run: |
-          make dependencies
+          make PYTHON_BIN=python3 dependencies
       - name: Run tests
         run: |
-          make unittest
+          make PYTHON_BIN=python3 unittest
 
   python3-rocky9:
     name: Run internal tests in default python3 on Rocky9
@@ -111,7 +111,7 @@ jobs:
           dnf install -y python3 python3-pip make
       - name: Setup environment
         run: |
-          make dependencies
+          make PYTHON_BIN=python3 dependencies
       - name: Run tests
         run: |
-          make unittest
+          make PYTHON_BIN=python3 unittest

.gitignore

@@ -1,4 +1,5 @@
 # Byte-compiled / optimized / DLL files
+__jinja__/
 __pycache__/
 *.py[cod]
 *$py.class
@@ -27,6 +28,9 @@ wheels/
 *.egg
 MANIFEST
 
+# Plugins
+mig/assets/migux/
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -182,6 +186,7 @@ mig/images/site-conf.js
 
 # self-test
 /envhelp/output/
+/envhelp/staging/
 /envhelp/venv/
 /envhelp/*.depends
 /envhelp/*.imageid

Makefile

@@ -10,15 +10,17 @@ FORMAT_ENFORCE_DIRS = state/
 FORMAT_EXCLUDE_REGEX = '.*'
 FORMAT_EXCLUDE_GLOB = '*'
 FORMAT_LINE_LENGTH = 80
-
 LOCAL_PYTHON_BIN = './envhelp/lpython'
+ROOT_DIR=$(realpath .)
+STAGING_DIR="$(ROOT_DIR)/envhelp/staging"
 
 ifdef PYTHON_BIN
 	LOCAL_PYTHON_BIN = $(PYTHON_BIN)
 else
 	PYTHON_BIN = './envhelp/python3'
 endif
 
+
 ifeq ($(ALLDEPS),1)
 	REQS_PATH = ./recommended.txt
 else
@@ -81,6 +83,7 @@ clean:
 distclean: clean
 	@rm -rf ./envhelp/venv
 	@rm -rf ./envhelp/output
+	@rm -rf ./envhelp/staging
 	@rm -rf ./tests/__pycache__
 	@rm -f ./tests/*.pyc
 
@@ -101,18 +104,36 @@ testconfig: ./envhelp/output/testconfs
 ./envhelp/output/testconfs:
 	@./envhelp/makeconfig test --docker
 	@./envhelp/makeconfig test
+	@echo "prime any configured templates"
+	@$(LOCAL_PYTHON_BIN) -m mig.lib.templates prime \
+		-c ./envhelp/output/testconfs-local/MiGserver.conf
 
 ifeq ($(MIG_ENV),'local')
 ./envhelp/local.depends: $(REQS_PATH) local-requirements.txt
 else
 ./envhelp/local.depends: $(REQS_PATH)
+endif
+ifeq ($(MIG_ENV),'local')
+	@echo "installing development dependencies"
+	@$(LOCAL_PYTHON_BIN) -m pip install \
+		-r local-requirements.txt
+	@echo "expanding local packages"
+	@$(LOCAL_PYTHON_BIN) ./envhelp/scripts/expand_sources.py
+	@echo "generating local package index"
+	@./envhelp/invoke dumb-pypi \
+		--package-list "$(STAGING_DIR)/.packages.lst" \
+		--packages-url='../..' \
+		--output-dir "$(STAGING_DIR)"
 endif
 	@echo "installing dependencies from $(REQS_PATH)"
 	@$(LOCAL_PYTHON_BIN) -m pip install -r $(REQS_PATH)
 ifeq ($(MIG_ENV),'local')
-	@echo ""
-	@echo "installing development dependencies"
-	@$(LOCAL_PYTHON_BIN) -m pip install -r local-requirements.txt
+	@echo "installing plugins"
+	@$(LOCAL_PYTHON_BIN) -m pip install \
+		-r ./mig/install/requirements/migux-requirements.txt \
+		--extra-index-url="file://$(STAGING_DIR)/simple"
+	@echo "running plugins postinstall"
+	@$(LOCAL_PYTHON_BIN) ./mig/install/postinstall/migux-postinstall
 endif
 	@touch ./envhelp/local.depends
 
@@ -121,4 +142,4 @@ endif
 	@/usr/bin/env python3 -m venv ./envhelp/venv
 	@rm -f ./envhelp/local.depends
 	@echo "upgrading venv pip as required for some dependencies"
-	@./envhelp/venv/bin/pip3 install --upgrade pip
+	@$(LOCAL_PYTHON_BIN) -m pip install --upgrade pip

envhelp/docker/Dockerfile.py3

@@ -15,4 +15,12 @@ WORKDIR /usr/src/app
 COPY requirements.txt local-requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt
 
+# insert dist files for our local packages
+RUN mkdir -p ./envhelp
+COPY ./envhelp/staging ./envhelp/staging
+COPY ./mig/install/requirements ./envhelp/requirements
+RUN pip install --no-cache-dir \
+                --index-url=file:///usr/src/app/envhelp/staging/simple \
+                 -r ./envhelp/requirements/migux-requirements.txt
+
 CMD [ "python", "--version" ]

envhelp/dpython

@@ -75,7 +75,12 @@ IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "")
 # explicit statement of the uid for use within the container for clarity
 CONTAINER_USER_UID=$LOCAL_USER_UID
 
-IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_FILE" . -q --build-arg "pyver=$PYVER" --build-arg "CONTAINER_USER_UID=$CONTAINER_USER_UID")
+IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_FILE" . -q \
+                                                  --build-arg "pyver=$PYVER" \
+                                                  --build-arg "CONTAINER_USER_UID=$CONTAINER_USER_UID" \
+                                                  --build-arg "EXTRA_INDEX_URL=$EXTRA_INDEX_URL" \
+)
+
 if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then
     echo "rebuilt for changes"
 

envhelp/invoke

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# --- BEGIN_HEADER ---
+#
+# python3 - wrapper to invoke a local python3 virtual environment
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+set -e
+
+SCRIPT_PATH=$(realpath "$0")
+SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH")
+MIG_BASE=$(realpath "$SCRIPT_BASE/..")
+
+PYTHON_BIN=${PYTHON_BIN:-"$SCRIPT_BASE/venv/bin/python3"}
+
+if [ "$1" = "" ]; then
+    echo "No binary requested"
+    exit 1
+fi
+
+BIN_PATH=$("$PYTHON_BIN" "$SCRIPT_BASE/which.py" $1)
+if [ "$BIN_PATH" = "" ]; then
+    echo "Unable to find the binary to invoke"
+    exit 1
+fi
+
+shift 1
+
+"$BIN_PATH" "$@"

envhelp/makeconfig.py

@@ -39,9 +39,9 @@
 
 sys.path.append(_LOCAL_MIG_BASE)
 
 from mig.shared.conf import get_configuration_object
 from mig.shared.install import MIG_BASE, generate_confs
 from mig.shared.useradm import _ensure_dirs_needed_for_userdb
 
 _LOCAL_ENVHELP_OUTPUT_DIR = os.path.join(_LOCAL_MIG_BASE, "envhelp/output")
 _MAKECONFIG_ALLOWED = ["local", "test"]
@@ -82,6 +82,8 @@
         'mig_code': os.path.join(conf_dir_path, 'mig'),
         'mig_certs': os.path.join(conf_dir_path, 'certs'),
         'mig_state': os.path.join(conf_dir_path, 'state'),
+        'admin_email': 'admin@example.com',
+        'support_email': 'support@example.com',
     })
 
     if is_docker:

envhelp/scripts/expand_sources.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import urllib.request
+import zipfile
+
+
+SCRIPT_DIR = os.path.dirname(__file__)
+ROOT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, '../..'))
+STAGING_DIR = os.path.join(ROOT_DIR, 'envhelp/staging')
+SOURCE_URL_BASE = 'http://github.com/ucphhpc'
+
+PACKAGE_REPOS_TO_BRANCH = {
+    'migux': {
+        'repository': 'migrid-ux',
+        'branch': 'next',
+    }
+}
+
+
+def download_url(url, output_file, use_wget=False):
+    if use_wget:
+        download_url_wget(url, output_file)
+    else:
+        urllib.request.urlretrieve(url, output_file)
+
+
+def download_url_wget(url, output_file):
+    subprocess.run(["wget", "-O", output_file, url], stderr=subprocess.DEVNULL)
+
+
+def main(argv):
+    use_wget = '--use-wget' in argv
+
+    # start with an entirely clean state
+    shutil.rmtree(STAGING_DIR, ignore_errors=True)
+
+    # make the package directory
+    os.makedirs(STAGING_DIR)
+
+    # make the temporary download directory
+    staging_dir = os.path.join(STAGING_DIR, ".download")
+    os.mkdir(staging_dir)
+
+    for package_name, package_dict  in PACKAGE_REPOS_TO_BRANCH.items():
+        package_repo = package_dict['repository']
+        branch_name = package_dict['branch']
+
+        package_repo_with_branch = "%s-%s" % (package_repo, branch_name)
+        package_dir_staging = os.path.join(staging_dir,  package_repo_with_branch)
+        downloaded_archive_file = os.path.join(staging_dir, "_%s.zip" % (package_repo,))
+
+        # grab an archive containing installable package files
+        package_url = os.path.join(SOURCE_URL_BASE, package_repo, 'archive/refs/heads', "%s.zip" % (branch_name,))
+        download_url(package_url, downloaded_archive_file, use_wget=use_wget)
+
+        # extract it
+        archive = zipfile.ZipFile(downloaded_archive_file)
+        archive.extractall(staging_dir)
+
+        # find the raw package files
+        packages_glob = os.path.join(package_dir_staging, "dist/%s-*" % (package_name,))
+        packages_paths = glob.glob(packages_glob)
+
+        # copy the raw package files into a common packages directory
+        for package_path in packages_paths:
+            package_file_name = os.path.basename(package_path)
+            target_path_path = os.path.join(STAGING_DIR, package_file_name)
+            shutil.copyfile(package_path, target_path_path)
+
+        # remove all staged files
+        shutil.rmtree(staging_dir)
+
+    packages_list = os.path.join(STAGING_DIR, ".packages.lst")
+
+    # write a listing of the available package files alongside them
+    with open(packages_list, "w") as outfile:
+        package_names_only = [file_name for file_name in os.listdir(STAGING_DIR)
+                                         if not file_name.startswith('.')]
+        package_names_only.sort()
+        print(*package_names_only, sep='\n', file=outfile)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

envhelp/which.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# python3 - wrapper to invoke a local python3 virtual environment
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+"""Locate a binary via the active python interpreter."""
+
+import os
+import shutil
+import sys
+
+ROOT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
+VENV_BIN_DIR = os.path.join(ROOT_DIR, 'envhelp/venv/bin')
+
+
+def main(args):
+    if len(args) != 1:
+        return 1
+
+    location = shutil.which(args[0])
+    if location is not None:
+        print(location)
+        return 0
+
+    # locally in development we do not require activating the venv and thus
+    # it is possible the binary is installed but not path visible - try the
+    # local venv as a fallback path to try to catch this
+    location = os.path.join(VENV_BIN_DIR, args[0])
+    if os.path.exists(location):
+        print(location)
+        return 0
+
+    return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

local-requirements.txt

@@ -4,6 +4,7 @@
 # We only need autopep8 on py 3 as it's used in 'make fmt' (with py3)
 autopep8;python_version >= "3"
 black
+dumb-pypi
 isort
 # We need paramiko for the ssh unit tests
 # NOTE: paramiko-3.0.0 dropped python2 and python3.6 support