DO NOT MERGE: Jl/parameter exposure in yaml

src/aosm/azext_aosm/build_processors/helm_chart_processor.py

@@ -5,7 +5,7 @@
 
 import json
 import re
-from typing import Any, Dict, List, Set, Tuple
+from typing import Any, Dict, List, Set, Tuple, Optional
 
 from knack.log import get_logger
 
@@ -400,3 +400,76 @@ def _remove_key_from_dict(self, dictionary: Dict[str, Any], path: str) -> None:
             return None  # Key removed
         # Otherwise, recursively call the function on the sub-dictionary
         return self._remove_key_from_dict(dictionary[keys[0]], ".".join(keys[1:]))
+
+    def _generate_schema(
+        self,
+        deploy_params_schema: Dict[str, Any],
+        source_schema: Dict[str, Any],
+        default_values: Dict[str, Any],
+        param_prefix: Optional[str] = None,
+    ) -> None:
+        """
+        Generate the JSON schema for deployParameters. The NFDProcessor overrides this method to generate CGS for NSDs.
+
+        This method recursively generates the deployParameters schema for the input artifact by updating
+        the schema parameter.
+
+        Note there is no return value. The schema is passed by reference and modified in place.
+
+        Parameters:
+            deploy_params_schema:
+                The schema to be modified. On first call of this method,
+                it should contain any base nodes for the schema.
+                This schema is passed by reference and modified in place.
+                This property is defined by the CLI in the base processor.
+            source_schema:
+                The source schema from which the deploy parameters schema is generated.
+                E.g., for a Helm chart this may be the schema generated from the values.yaml file.
+                For an ARM template this will be the schema generated from the template's parameters
+            default_values:
+                The default values used to determine whether a parameter should be hardcoded or provided by the user.
+                Defined by the input artifact classes from the config provided by the user.
+                E.g. for helm charts this can be the default values file or the values.yaml file in the chart.
+            param_prefix:
+                The prefix to be added to the parameter name.
+                This is used for namespacing nested properties in the schema.
+                On first call to this method this should be None.
+        """
+        if "properties" not in source_schema.keys():
+            return
+        # Abbreviated 'prop' because 'property' is built in.
+
+        if self.expose_all_params:
+            # loop through
+            for prop, details in source_schema["properties"].items():
+                param_name = prop if not param_prefix else f"{param_prefix}_{prop}"
+                print(param_name)
+
+            # if hardcode nfdv, dont expose, else expose
+        # else:
+        else:
+            # loop through
+            for prop, details in source_schema["properties"].items():
+                param_name = prop if not param_prefix else f"{param_prefix}_{prop}"
+                print(param_name)
+
+                # 1. Required parameters (in 'required' array and no default given).
+                if (
+                    prop not in default_values
+                    and "required" in source_schema
+                    and prop in source_schema["required"]
+                ):
+                    # Note: we only recurse into objects, not arrays. For now, this is sufficient.
+                    if "properties" in details:
+                        self._generate_schema(
+                            deploy_params_schema,
+                            details,
+                            default_values={},  # In this branch property_key is not in defaults, so pass empty dict.
+                            param_prefix=param_name,
+                        )
+                    else:
+                        deploy_params_schema["required"].append(param_name)
+                        deploy_params_schema["properties"][param_name] = details
+            # if no default but required, expose
+            # if expose-cgs, expose
+

src/aosm/azext_aosm/inputs/helm_chart_input.py

@@ -18,7 +18,7 @@
 import yaml
 from knack.log import get_logger
 from ruamel.yaml.error import ReusedAnchorWarning
-
+from ruamel.yaml import CommentedMap, CommentedSeq
 from azext_aosm.common.exceptions import (
     DefaultValuesNotFoundError,
     MissingChartDependencyError,
@@ -30,7 +30,7 @@
 from azext_aosm.inputs.base_input import BaseInput
 
 logger = get_logger(__name__)
-yaml_processor = ruamel.yaml.YAML(typ="safe", pure=True)
+yaml_processor = ruamel.yaml.YAML(typ="rt", pure=True)
 warnings.simplefilter("ignore", ReusedAnchorWarning)
 
 
@@ -422,6 +422,91 @@ def _get_metadata(self) -> HelmChartMetadata:
             dependencies=dependencies,
         )
 
+    def get_yaml_values_and_comments(self, data):
+        """
+        Recursively processes YAML file and creates a new data structure 
+        that mirrors the original one, but each value is replaced with a dictionary containing the value 
+        and its associated comment (if it exists and is one of our keywords).
+
+        For example, in the values.yaml:
+
+        image:
+            repository: overwriteme # expose-cgs
+            tag: stable
+
+        the object created looks like: 
+
+        "image": {
+            "value": {
+                "repository": {
+                    "value": "overwriteme",
+                    "comment": "expose-cgs"
+                },
+                "tag": {
+                    "value": "stable",
+                    "comment": null
+                },
+
+        The function handles CommentedMap (similar to a dictionary) and CommentedSeq (similar to a list) 
+        data types from the ruamel.yaml library, which preserve comments in the YAML file.
+
+        Args:
+            data (CommentedMap or CommentedSeq): The data structure loaded from the YAML file.
+
+        Returns:
+            dict or list: A new data structure that mirrors the values.yaml, but each value is replaced 
+            with a dictionary containing the value and its associated comment (if it exists). The type of 
+            the returned data structure matches the type of the input data (i.e., a dictionary for 
+            CommentedMap and a list for CommentedSeq).
+        """
+
+        if isinstance(data, CommentedMap):
+            processed_yaml_data = {}
+            for key, value in data.items():
+                comment = None
+                # Check if the key has an associated comment
+                if key in data.ca.items:
+                    comment_value = data.ca.items[key]
+                    # Comments are stored as
+                    # [None, None, CommentToken('Test\n', line: 4, col: 14), None] so we look at the 3rd element
+                    if comment_value and comment_value[2]:
+                        if "expose-cgs" in comment_value[2].value:
+                            comment = "expose-cgs"
+                        elif "hardcode-nfdv" in comment_value[2].value:
+                            comment = "hardcode-nfdv"
+                        else:
+                            comment = None
+                # Add the value and its comment to the new dictionary
+                if isinstance(value, (CommentedMap, CommentedSeq)):
+                    processed_yaml_data[key] = {'value': self.get_yaml_values_and_comments(value), 'comment': comment}
+                else:
+                    processed_yaml_data[key] = {'value': value, 'comment': comment}
+            return processed_yaml_data
+
+        elif isinstance(data, CommentedSeq):
+            processed_yaml_data = []
+            for index, item in enumerate(data):
+                comment = None
+                # Check if the index has an associated comment
+                if index in data.ca.items:
+                    comment_value = data.ca.items[index]
+
+                    if comment_value and comment_value[2]:
+                        if "expose-cgs" in comment_value[2].value:
+                            comment = "expose-cgs"
+                        elif "hardcode-nfdv" in comment_value[2].value:
+                            comment = "hardcode-nfdv"
+                        else:
+                            comment = None
+                # Add the item and its comment to the new list
+                if isinstance(item, (CommentedMap, CommentedSeq)):
+                    processed_yaml_data.append({'value': self.get_yaml_values_and_comments(item), 'comment': comment})
+                else:
+                    processed_yaml_data.append({'value': item, 'comment': comment})
+            return processed_yaml_data
+        else:
+            return data
+
     def _read_values_yaml_from_chart(self) -> Dict[str, Any]:
         """
         Reads the values.yaml file in the Helm chart directory.
@@ -435,7 +520,10 @@ def _read_values_yaml_from_chart(self) -> Dict[str, Any]:
             if file.name.endswith(("values.yaml", "values.yml")):
                 with file.open(encoding="UTF-8") as f:
                     content = yaml_processor.load(f)
-                return content
+                # print(content)
+                test = self.get_yaml_values_and_comments(content)
+                print("OUTPUT", json.dumps(test, indent=4))
+                return self.get_yaml_values_and_comments(content)
 
         logger.error(
             "values.yaml|yml file not found in Helm chart '%s'", self.chart_path