Introduce support for jinja2-based templates.

This addds the necessary logic both to maintain a store of templates
and makiung them available for serving within any page via a new
template object type.

Make sure there is a separation of the test templates from the mainline
stuff by wiring a TEMPLATES section into Configuration and reading the
necessary paths via the loaded configuration. Doing so allows both
source location and the cache directory to be optionally specified via
the main ini file and thus are runtime accessible without hard-coding.
Use this facility within the tests to adjust the paths.

Author: albu-diku

.gitignore

@@ -1,4 +1,5 @@
 # Byte-compiled / optimized / DLL files
+__jinja__/
 __pycache__/
 *.py[cod]
 *$py.class

mig/assets/templates/.gitkeep

@@ -0,0 +1,138 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+import errno
+from collections import ChainMap
+from jinja2 import meta as jinja2_meta, select_autoescape, Environment, \
+    FileSystemLoader, FileSystemBytecodeCache
+import os
+import weakref
+
+from mig.shared.compat import PY2
+from mig.shared.defaults import MIG_BASE
+
+
+class _BonundTemplate:
+    def __init__(self, template, template_args):
+        self.tmpl = template
+        self.args = template_args
+
+
+    def render(self):
+        return self.tmpl.render(**self.args)
+
+
+class _FormatContext:
+    def __init__(self, configuration):
+        self.output_format = None
+        self.configuration = configuration
+        self.script_map = {}
+        self.style_map = {}
+
+    def __getitem__(self, key):
+        return self.__dict__[key]
+
+    def __iter__(self):
+        return iter(self.__dict__)
+
+    def extend(self, template, template_args):
+        return _BonundTemplate(template, ChainMap(template_args, self))
+
+
+class TemplateStore:
+    def __init__(self, template_dirs, cache_dir=None, extra_globals=None):
+        assert cache_dir is not None
+
+        self._cache_dir = cache_dir
+        self._template_globals = extra_globals
+        self._template_environment = Environment(
+            loader=FileSystemLoader(template_dirs),
+            bytecode_cache=FileSystemBytecodeCache(cache_dir, '%s'),
+            autoescape=select_autoescape()
+        )
+
+    @property
+    def cache_dir(self):
+        return self._cache_dir
+
+    @property
+    def context(self):
+        return self._template_globals
+
+    def _get_template(self, template_fqname):
+        return self._template_environment.get_template(template_fqname)
+
+    def grab_template(self, template_name, template_group, output_format, template_globals=None, **kwargs):
+        template_fqname = "%s_%s.%s.jinja" % (
+            template_group, template_name, output_format)
+        return self._template_environment.get_template(template_fqname, globals=template_globals)
+
+    def list_templates(self):
+        return [t for t in self._template_environment.list_templates() if t.endswith('.jinja')]
+
+    def extract_variables(self, template_fqname):
+        template = self._template_environment.get_template(template_fqname)
+        with open(template.filename) as f:
+            template_source = f.read()
+        ast = self._template_environment.parse(template_source)
+        return jinja2_meta.find_undeclared_variables(ast)
+
+    @staticmethod
+    def populated(template_dirs, cache_dir=None, context=None):
+        assert cache_dir is not None
+
+        try:
+            os.mkdir(cache_dir)
+        except OSError as direxc:
+            if direxc.errno != errno.EEXIST:  # FileExistsError
+                raise
+
+        store = TemplateStore(
+            template_dirs, cache_dir=cache_dir, extra_globals=context)
+
+        for template_fqname in store.list_templates():
+            store._get_template(template_fqname)
+
+        return store
+
+
+def init_global_templates(configuration):
+    template_division = configuration.division(section_name="TEMPLATES")
+
+    _context = configuration.context()
+
+    try:
+        return configuration.context(namespace='templates')
+    except KeyError as exc:
+        pass
+
+    store = TemplateStore.populated(
+        [template_division.source_dir],
+        cache_dir=template_division.cache_dir,
+        context=_FormatContext(configuration)
+    )
+    return configuration.context_set(store, namespace='templates')

mig/lib/__init__.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+from types import SimpleNamespace
+import os
+import sys
+
+from mig.lib.templates import init_global_templates
+from mig.shared.conf import get_configuration_object
+
+
+def warn(message):
+    print(message, file=sys.stderr, flush=True)
+
+
+def main(args, _print=print):
+    configuration = get_configuration_object(config_file=args.config_file)
+    template_store = init_global_templates(configuration)
+
+    command = args.command
+    if command == 'show':
+        print(template_store.list_templates())
+    elif command == 'prime':
+        try:
+            os.mkdir(template_store.cache_dir)
+        except FileExistsError:
+            pass
+
+        for template_fqname in template_store.list_templates():
+            template_store._get_template(template_fqname)
+    elif command == 'vars':
+        for template_ref in template_store.list_templates():
+            _print("<%s>" % (template_ref,))
+            for var in template_store.extract_variables(template_ref):
+                _print("  {{%s}}" % (var,))
+            _print("</%s>" % (template_ref,))
+    else:
+        raise RuntimeError("unknown command: %s" % (command,))
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', dest='config_file', default=None)
+    parser.add_argument('command')
+    args = parser.parse_args()
+
+    try:
+        main(args)
+        sys.exit(0)
+    except Exception as exc:
+        warn(str(exc))
+        sys.exit(1)

mig/lib/templates/__init__.py

@@ -44,6 +44,7 @@
 import socket
 import sys
 import time
+from types import SimpleNamespace
 
 # Init future py2/3 compatibility helpers
 
@@ -60,7 +61,9 @@
 # NOTE: protect migrid import from autopep8 reordering
 try:
     from mig.shared.base import force_native_str
-    from mig.shared.defaults import CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
+    from mig.shared.base import force_native_str
+    from mig.shared.defaults import MIG_BASE, \
+        CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
         CSRF_FULL, POLICY_NONE, POLICY_WEAK, POLICY_MEDIUM, POLICY_HIGH, \
         POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, cert_field_order, \
         default_css_filename, keyword_any, keyword_auto, keyword_all, \
@@ -180,14 +183,15 @@ def expand_external_sources(logger, val):
     return expanded
 
 
-def fix_missing(config_file, verbose=True):
-    """Add missing configuration options - used by checkconf script"""
+_MARKER_ADMIN_EMAIL = object()
+_MARKER_FQDN = object()
+_MARKER_USER = object()
 
-    config = ConfigParser()
-    config.read([config_file])
 
-    fqdn = socket.getfqdn()
-    user = os.environ['USER']
+def _generate_fix_missing_definitions():
+    fqdn = _MARKER_FQDN
+    user = _MARKER_USER
+
     global_section = {
         'enable_server_dist': False,
         'auto_add_cert_user': False,
@@ -200,7 +204,7 @@ def fix_missing(config_file, verbose=True):
         'auto_add_filter_fields': '',
         'server_fqdn': fqdn,
         'support_email': '',
-        'admin_email': '%s@%s' % (user, fqdn),
+        'admin_email': _MARKER_ADMIN_EMAIL,
         'admin_list': '',
         'ca_fqdn': '',
         'ca_smtp': '',
@@ -399,23 +403,53 @@ def fix_missing(config_file, verbose=True):
     quota_section = {'backend': 'lustre',
                      'user_limit': 1024**4,
                      'vgrid_limit': 1024**4}
-    defaults = {
+
+    default_template_source_dir = os.path.join(MIG_BASE, 'mig/assets/templates')
+    default_template_cache_dir = os.path.join(default_template_source_dir, '__jinja__')
+
+    templates_section = {'source_dir': default_template_source_dir,
+                         'cache_dir': default_template_cache_dir}
+
+    return {
         'GLOBAL': global_section,
         'SCHEDULER': scheduler_section,
         'MONITOR': monitor_section,
         'SETTINGS': settings_section,
         'FEASIBILITY': feasibility_section,
         'WORKFLOWS': workflows_section,
         'QUOTA': quota_section,
+        'TEMPLATES': templates_section,
     }
-    for section in defaults:
+
+
+_FIX_MISSING_DEFINITIONS = _generate_fix_missing_definitions()
+
+
+def fix_missing():
+    """Add missing configuration options  used by checkconf script"""
+
+    fqdn = socket.getfqdn()
+    user = os.environ['USER']
+
+    _marker_substitutions = {
+        _MARKER_ADMIN_EMAIL: "%s@%s" % (user, fqdn),
+        _MARKER_FQDN: fqdn,
+        _MARKER_USER: user,
+    }
+
+    config = ConfigParser()
+    config.read([config_file])
+
+    modified = False
+    for (section, settings) in _FIX_MISSING_DEFINITIONS.items():
         if not section in config.sections():
             config.add_section(section)
 
-    modified = False
-    for (section, settings) in defaults.items():
         for (option, value) in settings.items():
             if not config.has_option(section, option):
+                if value in _marker_substitutions:
+                    value = _marker_substitutions[value]
+
                 if verbose:
                     print('setting %s->%s to %s' % (section, option,
                                                     value))
@@ -435,6 +469,11 @@ def fix_missing(config_file, verbose=True):
         fd.close()
 
 
+def _only_valid_for_section(candididates_dict, /, section_name):
+    section_defaults = dict(_FIX_MISSING_DEFINITIONS[section_name])
+    return {k: v for k, v in candididates_dict.items() if k in section_defaults}
+
+
 class NativeConfigParser(ConfigParser):
     """Wraps configparser.ConfigParser to force get method to return native
     string instead of always returning unicode.
@@ -767,6 +806,9 @@ def __init__(self, config_file, verbose=False, skip_log=False,
         self.auth_logger_obj = None
         self.gdp_logger_obj = None
 
+        # structured
+        self._divisions = {}
+
         configuration_options = copy.deepcopy(_CONFIGURATION_DEFAULTS)
 
         for k, v in configuration_options.items():
@@ -1038,6 +1080,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
                 pass
             raise Exception('Failed to parse configuration: %s' % err)
 
+        # handle structured sections
+        self.apply_loaded_config_to_division(config, section_name='TEMPLATES')
+
         # Remaining options in order of importance - i.e. options needed for
         # later parsing must be parsed and set first.
 
@@ -2844,6 +2889,31 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
                          % keyword_all)
             self.site_twofactor_mandatory_protos = [keyword_all]
 
+    def apply_loaded_config_to_division(self, loaded_config, /, section_name):
+        active_division = self.division(section_name=section_name)
+
+        try:
+            templates_candidates = dict(loaded_config[section_name])
+            templates_overrides = _only_valid_for_section(templates_candidates, section_name=section_name)
+        except KeyError:
+            templates_overrides = {}
+
+        active_division.__dict__.update(templates_overrides)
+
+        self._divisions[section_name] = active_division
+
+
+    def division(self, /, section_name):
+        if section_name not in _FIX_MISSING_DEFINITIONS:
+            raise NotImplementedError()
+
+        try:
+            return self._divisions[section_name]
+        except KeyError:
+            new_division = SimpleNamespace(**_FIX_MISSING_DEFINITIONS[section_name])
+            self._divisions[section_name] = new_division
+            return new_division
+
     def parse_peers(self, peerfile):
 
         # read peer information from peerfile