Feature branch sync - pub/q2 upgrade to staging

.github/workflows/ansible-lint.yml

@@ -10,6 +10,7 @@ on:
       - pub/q2_dev
       - pub/telemetry
       - pub/q2_upgrade
+      - pub/q2_ansible
 
 jobs:
   build:

.github/workflows/pylint.yml

@@ -10,6 +10,7 @@ on:
       - pub/q2_dev
       - pub/telemetry
       - pub/q2_upgrade
+      - pub/q2_ansible
 
 jobs:
   build:

ansible.cfg

@@ -7,8 +7,9 @@ forks = 5
 timeout = 180
 executable = /bin/bash
 display_skipped_hosts = false
-library = discovery/library:common/library/modules
-#inventory = /opt/omnia/omnia_inventory/cluster_layout
+deprecation_warnings = false
+show_task_path_on_failure = false
+library = common/library/modules
 module_utils = common/library/module_utils
 
 [persistent_connection]

build_image_aarch64/ansible.cfg

@@ -5,6 +5,9 @@ host_key_checking = false
 forks = 5
 timeout = 180
 executable = /bin/bash
+interpreter_python = /usr/bin/python3
+deprecation_warnings = false
+show_task_path_on_failure = false
 library = ../common/library/modules
 module_utils = ../common/library/module_utils
 

build_image_aarch64/build_image_aarch64.yml

@@ -24,7 +24,7 @@
     - name: Set dynamic run tags including 'build_aarch_image'
       when: not config_file_status | default(false) | bool
       ansible.builtin.set_fact:
-        omnia_run_tags: "{{ (ansible_run_tags | default([]) + ['build_aarch_image']) | unique }}"
+        omnia_run_tags: "{{ (ansible_run_tags | default([]) | list + ['build_aarch_image']) | unique }}"
         cacheable: true
 
 - name: Invoke validate_config.yml to perform L1 and L2 validations with build_image tag

build_image_aarch64/roles/image_creation/tasks/build_base_image.yml

@@ -76,7 +76,7 @@
 
     - name: Verify the aarch64 base osimage in registry
       ansible.builtin.command:
-        cmd: "regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
+        cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
       delegate_to: "{{ aarch64_build_host }}"
       connection: ssh
       changed_when: false

build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml

@@ -102,7 +102,7 @@
 
     - name: Verify aarch64 compute osimages in registry
       ansible.builtin.command:
-        cmd: "regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
+        cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
       delegate_to: "{{ aarch64_build_host }}"
       connection: ssh
       changed_when: false

build_image_x86_64/ansible.cfg

@@ -5,6 +5,9 @@ host_key_checking = false
 forks = 5
 timeout = 180
 executable = /bin/bash
+interpreter_python = /usr/bin/python3
+deprecation_warnings = false
+show_task_path_on_failure = false
 library = ../common/library/modules
 module_utils = ../common/library/module_utils
 

build_image_x86_64/build_image_x86_64.yml

@@ -24,7 +24,7 @@
     - name: Set dynamic run tags including 'build_image'
       when: not config_file_status | default(false) | bool
       ansible.builtin.set_fact:
-        omnia_run_tags: "{{ (ansible_run_tags | default([]) + ['build_image']) | unique }}"
+        omnia_run_tags: "{{ (ansible_run_tags | default([]) | list + ['build_image']) | unique }}"
         cacheable: true
 
 - name: Invoke validate_config.yml to perform L1 and L2 validations with build_image tag

build_image_x86_64/roles/image_creation/tasks/build_base_image.yml

@@ -66,7 +66,7 @@
 
     - name: Verify the x86_64 base osimage in registry
       ansible.builtin.command:
-        cmd: "regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
+        cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
       changed_when: false
       register: verify_base_osimage
 

build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml

@@ -94,7 +94,7 @@
 
     - name: Verify x86_64 compute osimages in registry
       ansible.builtin.command:
-        cmd: "regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
+        cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000"
       changed_when: false
       register: verify_compute_osimages
 

build_stream/core/catalog/CATALOG_GENERATOR.md

@@ -0,0 +1,106 @@
+# Catalog generator
+
+This directory contains utility tools for catalog generation and validation.
+
+## Tools
+
+### 1. generate_catalog_examples.py
+
+Generates example catalog JSON files from input configuration by cycling through different mapping/software_config combinations.
+
+**Location:** `build_stream/generate_catalog_examples.py`
+
+**Usage:**
+```bash
+cd /omnia/build_stream
+python3 generate_catalog_examples.py --base-dir /omnia/input/project_default/
+```
+
+**What it does:**
+- Copies mapping files from `examples/catalog/mapping_file_software_config/` to the input directory
+- Generates catalogs for each mapping variant (slurm-only, nfs-provisioner, etc.)
+- Outputs generated catalogs to `examples/catalog/` directory
+- Provides a summary of packages and layers generated
+
+**Generated catalogs:**
+- `catalog_rhel_aarch64_with_slurm_only.json`
+- `catalog_rhel_x86_64_with_slurm_only.json`
+- `catalog_rhel_with_nfs_provisioner.json`
+- `catalog_rhel_x86_64.json`
+- `catalog_rhel.json`
+
+---
+
+### 2. diff_input_configs.py
+
+Compares two input directories (expected vs actual) and reports per-file, per-cluster package differences.
+This can be used independantly or after running the catalog generator to check the differences.
+
+**Location:** `build_stream/core/catalog/tests/diff_input_configs.py`
+
+**Usage:**
+```bash
+cd /omnia/build_stream/core/catalog/tests
+python3 diff_input_configs.py \
+    --expected /omnia/input \
+    --actual   /tmp/adapter_output_test/input
+```
+
+**Optional arguments:**
+- `--file-level`: Compare packages at file level (flatten all clusters) instead of per-cluster
+- `--report <path>`: Write a human-readable table report to the given file
+- `--pxe-mapping <path>`: Path to PXE mapping CSV file for information display
+- `--catalog <path>`: Path to catalog file for information display
+
+**What it does:**
+1. Compares `software_config.json` (softwares list and versions)
+2. Walks `config/<arch>/<os>/<ver>/*.json` in both directories
+3. For each matching JSON, compares packages per cluster section
+4. Reports missing files, extra files, and per-cluster diffs
+5. Handles versioned filenames (e.g., `service_k8s_v1.35.1.json` matches `service_k8s.json`)
+6. Ignores common package extraction and `_first` cluster merging artifacts
+
+**Programmatic usage (for tests):**
+```python
+from diff_input_configs import run_diff_for_test
+
+passed, issue_count, report_path = run_diff_for_test(
+    expected_dir="/path/to/expected",
+    actual_dir="/path/to/actual",
+    report_file="/path/to/report.txt"  # optional, uses temp file if not provided
+)
+# Returns: (passed: bool, issue_count: int, report_path: str)
+```
+
+**Exit codes:**
+- `0`: No differences found
+- `1`: Differences found
+
+---
+
+### 3. test_catalog_diff_regression.py
+
+Regression test suite that validates catalog generation and adapter policy output.
+
+**Location:** `build_stream/core/catalog/tests/test_catalog_diff_regression.py`
+
+**Core idea:**
+Validates the end-to-end flow: catalog → adapter policy → input configs, ensuring the generated output matches the expected input configuration files.
+
+**Steps:**
+1. Loads example catalog (`catalog_rhel.json`)
+2. Runs generator to create root JSONs (functional_layer.json, infrastructure.json, etc.)
+3. Runs adapter policy to generate input configs from root JSONs
+4. Uses `diff_input_configs.py` to compare generated output with expected input configs
+5. Validates functional layers match PXE mapping expectations
+6. Checks specific package routing and architecture constraints
+
+**Usage:**
+```bash
+cd /omnia/build_stream/core/catalog/tests
+python3 -m pytest test_catalog_diff_regression.py -v
+```
+
+**Test classes:**
+- `TestAdapterDiffReport`: Verifies adapter output matches expected configs using diff tool
+- `TestCatalogFunctionalLayers`: Validates functional layers against PXE mapping and architecture constraints

build_stream/core/catalog/tests/test_generate_catalog_id.py

@@ -0,0 +1,84 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import generate_catalog
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+from generate_catalog import _generate_human_readable_id
+
+class TestGenerateHumanReadableId(unittest.TestCase):
+    def setUp(self):
+        self.used_ids = set()
+
+    def test_basic_names(self):
+        # Should remain unchanged
+        self.assertEqual(_generate_human_readable_id("apptainer", "rpm", None, self.used_ids), "apptainer")
+        self.assertEqual(_generate_human_readable_id("device-mapper-multipath", "rpm", None, self.used_ids), "device-mapper-multipath")
+
+    def test_version_in_name_exact(self):
+        # Should strip exact version suffix
+        self.assertEqual(_generate_human_readable_id("external-snapshotter-v8.4.0", "git", "v8.4.0", self.used_ids), "external-snapshotter")
+
+    def test_version_in_name_v_prefixed(self):
+        # Should strip if 'v' prefix is in name but not in version
+        self.assertEqual(_generate_human_readable_id("app-v1.0.0", "rpm", "1.0.0", self.used_ids), "app")
+
+    def test_version_in_name_dots_replaced(self):
+        # Should strip if dots are replaced by hyphens
+        self.assertEqual(_generate_human_readable_id("helm-charts-2-16-0", "git", "2.16.0", self.used_ids), "helm-charts")
+
+    def test_pip_module_format(self):
+        # PyMySQL==1.1.2 -> PyMySQL
+        self.assertEqual(_generate_human_readable_id("PyMySQL==1.1.2", "pip_module", None, self.used_ids), "PyMySQL")
+
+    def test_regex_fallback_no_version(self):
+        # Regex should strip the version even without explicit pkg_version
+        self.assertEqual(_generate_human_readable_id("calico-v3.31.4", "manifest", None, self.used_ids), "calico")
+        self.assertEqual(_generate_human_readable_id("cert-manager-v1-10-0", "tarball", None, self.used_ids), "cert-manager")
+        self.assertEqual(_generate_human_readable_id("helm-v3-20-1-amd64", "tarball", None, self.used_ids), "helm-amd64")
+        self.assertEqual(_generate_human_readable_id("helm-v3-20-1-chart", "tarball", None, self.used_ids), "helm-chart")
+        self.assertEqual(_generate_human_readable_id("helm-v3-20-1-anything-suffixed", "tarball", None, self.used_ids), "helm-anything-suffixed")
+        self.assertEqual(_generate_human_readable_id("metallb-native-v0-15-3", "manifest", None, self.used_ids), "metallb-native")
+        self.assertEqual(_generate_human_readable_id("strimzi-kafka-operator-helm-3-chart-0-48-0", "tarball", None, self.used_ids), "strimzi-kafka-operator-helm-3-chart")
+        self.assertEqual(_generate_human_readable_id("victoria-metrics-operator-0-59-3", "tarball", None, self.used_ids), "victoria-metrics-operator")
+        self.assertEqual(_generate_human_readable_id("python3-PyMySQL-1.1.2", "rpm", None, self.used_ids), "python3-PyMySQL")
+        self.assertEqual(_generate_human_readable_id("nfs-subdir-external-provisioner-4-0-18", "tarball", None, self.used_ids), "nfs-subdir-external-provisioner")
+
+    def test_docker_image_without_tag_in_name(self):
+        # Docker images usually don't have the tag in the name field, just the image path
+        self.assertEqual(_generate_human_readable_id("docker.io/library/python", "image", "3.12-slim", self.used_ids), "docker.io/library/python")
+
+    def test_collision_handling(self):
+        # First call gets the base name
+        id1 = _generate_human_readable_id("calico", "rpm", None, self.used_ids)
+        self.assertEqual(id1, "calico")
+
+        # Second call with the same base name gets _1
+        id2 = _generate_human_readable_id("calico-v1.0.0", "tarball", None, self.used_ids)
+        self.assertEqual(id2, "calico_1")
+
+        # Third call gets _2
+        id3 = _generate_human_readable_id("calico-v2.0.0", "manifest", None, self.used_ids)
+        self.assertEqual(id3, "calico_2")
+
+        self.assertIn("calico", self.used_ids)
+        self.assertIn("calico_1", self.used_ids)
+        self.assertIn("calico_2", self.used_ids)
+
+if __name__ == "__main__":
+    unittest.main()

build_stream/generate_catalog.py

@@ -177,37 +177,43 @@ def _merge_package_entries(dst, src):
 def _generate_human_readable_id(pkg_name, pkg_type, pkg_version, used_ids):
     """Generate a human-readable package ID with collision handling.
 
-    Format: {sanitized_name}_{version}_{type}
+    Format: {name} (the version is stripped from the name)
     If collision occurs, append counter: {base_id}_{counter}
     """
     # Extract version from package name if present (e.g., PyMySQL==1.1.2)
     name_for_id = pkg_name
-    if '==' in pkg_name and pkg_type == 'pip_module':
-        # For pip modules, name==version format
+    if '==' in pkg_name:
         parts = pkg_name.split('==')
-        if len(parts) == 2:
-            name_for_id = parts[0]
-            if not pkg_version:
-                pkg_version = parts[1]
-    elif '-' in pkg_name and pkg_type == 'rpm' and not pkg_version:
-        # Some RPMs have version in name (e.g., python3-PyMySQL-1.1.2)
-        # Try to extract version if it looks like a version number
-        parts = pkg_name.rsplit('-', 1)
-        if len(parts) == 2 and re.match(r'^\d', parts[1]):
-            name_for_id = parts[0]
+        name_for_id = parts[0]
+        if not pkg_version:
             pkg_version = parts[1]
+
+    # Try exact match removal if version is known
+    if pkg_version and isinstance(pkg_version, str):
+        # match at the end with 'v' prefixed
+        if name_for_id.endswith('v' + pkg_version):
+            name_for_id = name_for_id[:-(len(pkg_version) + 1)]
+        # exact match at the end
+        elif name_for_id.endswith(pkg_version):
+            name_for_id = name_for_id[:-len(pkg_version)]
+        # match with dots replaced by hyphens
+        elif name_for_id.endswith(pkg_version.replace('.', '-')):
+            name_for_id = name_for_id[:-len(pkg_version)]
+
+    # Use regex to strip version-like suffixes for remaining cases
+    # Matches:
+    # 1. -v followed by digits/dots/hyphens (e.g. -v1.2.3, -v2)
+    # 2. - followed by multi-part digits (e.g. -2-16-0, -1.1.2)
+    # 3. - followed by single digit at the end (e.g. -3$)
+    # Preserves trailing non-version suffixes (e.g. -amd64, -chart)
+    version_regex = r'[-_](?:v\d+(?:[-.]\d+)*|\d+(?:[-.]\d+)+|\d+$)(?=[-_]|$)'
+    name_for_id = re.sub(version_regex, '', name_for_id)
 
-    # Sanitize package name: replace special chars with hyphens
-    sanitized = re.sub(r'[/@:.]', '-', name_for_id)
-    sanitized = re.sub(r'-+', '-', sanitized).strip('-')
-
-    # Truncate very long names to reasonable length
-    if len(sanitized) > 50:
-        sanitized = sanitized[:50]
+    # Clean up trailing separators
+    name_for_id = name_for_id.rstrip('-_')
 
     # Build base ID
-    version_part = pkg_version if pkg_version else 'na'
-    base_id = f"{sanitized}_{version_part}_{pkg_type}"
+    base_id = name_for_id
 
     # Handle collisions
     if base_id not in used_ids:

build_stream/requirements-dev.txt

@@ -11,5 +11,5 @@ httpx>=0.25.0
 
 # Code quality
 pylint>=3.0.0
-black>=23.0.0
+black>=26.5.0
 isort>=5.12.0

build_stream/requirements.txt

@@ -8,7 +8,7 @@ pydantic>=2.5.0
 
 # Authentication
 PyJWT>=2.8.0
-cryptography>=41.0.0
+cryptography>=48.0.0
 argon2-cffi>=23.1.0
 
 # Dependency injection

common/library/module_utils/discovery/standard_functions.py

@@ -82,7 +82,7 @@ def update_json(new_data, filepath):
     """
     if os.path.exists(filepath):
         # Load existing data
-        with open(filepath, 'r') as f:
+        with open(filepath, 'r', encoding='utf-8') as f:
             try:
                 existing_data = json.load(f)
             except json.JSONDecodeError:
@@ -94,5 +94,5 @@ def update_json(new_data, filepath):
     existing_data.update(new_data)
 
     # Write back to file
-    with open(filepath, 'w') as f:
+    with open(filepath, 'w', encoding='utf-8') as f:
         json.dump(existing_data, f, indent=2)