diff --git a/cmake/AsicConfigV3ConfigCli.cmake b/cmake/AsicConfigV3ConfigCli.cmake new file mode 100644 index 0000000000000..7c070d586d93b --- /dev/null +++ b/cmake/AsicConfigV3ConfigCli.cmake @@ -0,0 +1,37 @@ +# CMake to build libraries and binaries in fboss/lib/asic_config_v3 +# Data-driven ASIC config generation + +include(FBPythonBinary) + +set( + ASIC_CONFIG_V3_PY_SRCS + "fboss/lib/asic_config_v3/__init__.py" + "fboss/lib/asic_config_v3/base_generator.py" + "fboss/lib/asic_config_v3/gen.py" + "fboss/lib/asic_config_v3/generators/__init__.py" + "fboss/lib/platform_mapping_v2/asic_vendor_config.py" + "fboss/lib/platform_mapping_v2/gen.py" + "fboss/lib/platform_mapping_v2/helpers.py" + "fboss/lib/platform_mapping_v2/platform_mapping_v2.py" + "fboss/lib/platform_mapping_v2/port_profile_mapping.py" + "fboss/lib/platform_mapping_v2/profile_settings.py" + "fboss/lib/platform_mapping_v2/read_files_utils.py" + "fboss/lib/platform_mapping_v2/si_settings.py" + "fboss/lib/platform_mapping_v2/static_mapping.py" +) + +add_fb_thrift_python_executable( + fboss-asic-config-v3-gen + MAIN_MODULE fboss.lib.asic_config_v3.gen:generate_all_asic_configs + SOURCES ${ASIC_CONFIG_V3_PY_SRCS} + DEPENDS + platform_config_python + switch_config_python + transceiver_python + phy_python + platform_mapping_config_python + fboss_common_python + python-pyyaml::python-pyyaml +) + +install_fb_python_executable(fboss-asic-config-v3-gen) diff --git a/fboss/lib/asic_config_v2/run-helper.sh b/fboss/lib/asic_config_v2/run-helper.sh index 80ba38234ea30..02491b608cb6e 100755 --- a/fboss/lib/asic_config_v2/run-helper.sh +++ b/fboss/lib/asic_config_v2/run-helper.sh @@ -1,2 +1,5 @@ #!/bin/bash -python3 fboss/lib/oss/run-helper.py --target fboss-asic-config-gen "$@" +python3 fboss/lib/oss/run-helper.py \ + --target fboss-asic-config-gen.GEN_PY_EXE \ + --extra-cmake-defines='{"RANGE_V3_TESTS": "OFF"}' \ + "$@" diff --git a/fboss/lib/asic_config_v3/__init__.py b/fboss/lib/asic_config_v3/__init__.py new file mode 100644 index 0000000000000..35302f75f52a5 --- /dev/null +++ b/fboss/lib/asic_config_v3/__init__.py @@ -0,0 +1 @@ +# pyre-strict diff --git a/fboss/lib/asic_config_v3/base_generator.py b/fboss/lib/asic_config_v3/base_generator.py new file mode 100644 index 0000000000000..f151cfd37ce4a --- /dev/null +++ b/fboss/lib/asic_config_v3/base_generator.py @@ -0,0 +1,82 @@ +# pyre-strict + +import copy +import os +from abc import ABC, abstractmethod +from typing import Any + +# Resolve config paths relative to the repo root rather than this file so the +# generator can run from the same checkout layout the bundled getdeps build +# expects. +_FBOSS_DIR: str = os.getcwd() + "/fboss" +MODULE_DIR: str = f"{_FBOSS_DIR}/lib/asic_config_v3" + + +def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + """Recursively merge ``override`` on top of ``base``. + + Dict values present in both are merged recursively. For any other value + type (scalars, lists, or a type mismatch between the two sides) the + override replaces the base entry outright. Returns a new dict; the inputs + are not mutated. + """ + result = copy.deepcopy(base) + for key, ov_value in override.items(): + if ( + key in result + and isinstance(result[key], dict) + and isinstance(ov_value, dict) + ): + result[key] = _deep_merge(result[key], ov_value) + else: + result[key] = copy.deepcopy(ov_value) + return result + + +class BaseAsicConfigGenerator(ABC): + """Abstract base class for ASIC config generators. + + Subclasses implement ``generate()`` to produce vendor-specific output + such as YAML or JSON. + """ + + def __init__( + self, + platform_name: str, + variant: str, + platform_config: dict[str, Any], + ) -> None: + self.platform_name = platform_name + self.variant = variant + self.platform_config = platform_config + + vendor = platform_config.get("vendor") + asic = platform_config.get("asic") + if not vendor: + raise ValueError("platform asic_config.json must define 'vendor'") + if not asic: + raise ValueError("platform asic_config.json must define 'asic'") + self.asic_vendor: str = vendor + self.asic_name: str = asic + + # The platform JSON may declare a top-level ``defaults`` block inherited + # by every variant. The effective variant config is produced by deep- + # merging the variant-specific entries on top of ``defaults``. Dict + # values are merged recursively; scalars and lists are replaced. + defaults = platform_config.get("defaults", {}) + variant_override = platform_config.get("variants", {}).get(variant, {}) + self.variant_config: dict[str, Any] = _deep_merge(defaults, variant_override) + self.asic_config_params: dict[str, Any] = self.variant_config.get( + "asic_config_params", {} + ) + + @abstractmethod + def generate(self) -> str: + """Generate the complete ASIC config and return it as a string.""" + ... + + @property + @abstractmethod + def output_extension(self) -> str: + """File extension for the generated output (e.g. '.yml' or '.json').""" + ... diff --git a/fboss/lib/asic_config_v3/common/ocp_sai_common.json b/fboss/lib/asic_config_v3/common/ocp_sai_common.json new file mode 100644 index 0000000000000..c0af00ae2bd0b --- /dev/null +++ b/fboss/lib/asic_config_v3/common/ocp_sai_common.json @@ -0,0 +1,4 @@ +{ + "_comment": "OCP SAI standard settings (vendor-agnostic).", + "global": {} +} diff --git a/fboss/lib/asic_config_v3/gen.py b/fboss/lib/asic_config_v3/gen.py new file mode 100644 index 0000000000000..aa43a4586a796 --- /dev/null +++ b/fboss/lib/asic_config_v3/gen.py @@ -0,0 +1,109 @@ +# pyre-strict + +import json +import os +import sys + +from fboss.lib.asic_config_v3.base_generator import ( + BaseAsicConfigGenerator, + MODULE_DIR, +) + +OUTPUT_DIR: str = f"{MODULE_DIR}/generated_asic_configs" + +# Add a new (vendor, asic) entry when bringing up a new ASIC family. +_GENERATOR_REGISTRY: dict[tuple[str, str], type[BaseAsicConfigGenerator]] = {} + + +def get_generator( + platform_name: str, variant: str, platform_config: dict +) -> BaseAsicConfigGenerator: + """Instantiate the correct generator based on vendor and ASIC.""" + vendor = platform_config["vendor"] + asic = platform_config["asic"] + key = (vendor, asic) + generator_cls = _GENERATOR_REGISTRY.get(key) + if not generator_cls: + raise ValueError(f"No generator registered for vendor={vendor}, asic={asic}") + return generator_cls(platform_name, variant, platform_config) + + +def discover_platforms() -> dict: + """Return a mapping of platform name to platform config. + + Discovered by scanning ``platforms/*/asic_config.json``. + """ + platforms_dir = os.path.join(MODULE_DIR, "platforms") + platforms = {} + + if not os.path.exists(platforms_dir): + return platforms + + for platform_name in os.listdir(platforms_dir): + platform_path = os.path.join(platforms_dir, platform_name) + if not os.path.isdir(platform_path): + continue + + config_path = os.path.join(platform_path, "asic_config.json") + if not os.path.exists(config_path): + continue + + with open(config_path) as f: + platform_config = json.load(f) + platforms[platform_name] = platform_config + + return platforms + + +def generate_all_asic_configs() -> None: + """Generate ASIC configs for every discovered platform and variant.""" + os.makedirs(OUTPUT_DIR, exist_ok=True) + + for filename in os.listdir(OUTPUT_DIR): + if filename.endswith((".json", ".yml")): + os.remove(os.path.join(OUTPUT_DIR, filename)) + + platforms = discover_platforms() + + for platform_name, platform_config in platforms.items(): + vendor = platform_config.get("vendor", "") + asic = platform_config.get("asic", "") + + if (vendor, asic) not in _GENERATOR_REGISTRY: + print( + f"Skipping {platform_name} (no generator for vendor={vendor}, asic={asic})", + file=sys.stderr, + ) + continue + + variants = platform_config.get("variants", {}) + + for variant_name in variants: + print( + f"Generating ASIC config for {platform_name}/{variant_name}...", + file=sys.stderr, + ) + + try: + generator = get_generator(platform_name, variant_name, platform_config) + output = generator.generate() + + output_filename = ( + f"{platform_name}_{variant_name}{generator.output_extension}" + ) + output_path = os.path.join(OUTPUT_DIR, output_filename) + + print(f"Writing to {output_path}", file=sys.stderr) + with open(output_path, "w", encoding="utf-8") as f: + f.write(output) + + except Exception as e: + print( + f"Error generating config for {platform_name}/{variant_name}: {e}", + file=sys.stderr, + ) + raise + + +if __name__ == "__main__": + generate_all_asic_configs() diff --git a/fboss/lib/asic_config_v3/generators/__init__.py b/fboss/lib/asic_config_v3/generators/__init__.py new file mode 100644 index 0000000000000..35302f75f52a5 --- /dev/null +++ b/fboss/lib/asic_config_v3/generators/__init__.py @@ -0,0 +1 @@ +# pyre-strict diff --git a/fboss/lib/asic_config_v3/run-helper.sh b/fboss/lib/asic_config_v3/run-helper.sh new file mode 100755 index 0000000000000..d42d68d7ddb80 --- /dev/null +++ b/fboss/lib/asic_config_v3/run-helper.sh @@ -0,0 +1,5 @@ +#!/bin/bash +python3 fboss/lib/oss/run-helper.py \ + --target fboss-asic-config-v3-gen.GEN_PY_EXE \ + --extra-cmake-defines='{"RANGE_V3_TESTS": "OFF"}' \ + "$@" diff --git a/fboss/lib/oss/run-helper.py b/fboss/lib/oss/run-helper.py index d271185e3752a..574fb1597191a 100644 --- a/fboss/lib/oss/run-helper.py +++ b/fboss/lib/oss/run-helper.py @@ -13,25 +13,36 @@ """ import argparse +import json import os import subprocess import sys from pathlib import Path -from typing import List, Tuple -def get_command_line_args() -> Tuple[str, List[str]]: +def get_command_line_args() -> tuple[str, dict[str, str], list[str]]: parser = argparse.ArgumentParser( description="OSS FBOSS build and run helper script." ) parser.add_argument("--target", type=str, required=True, help="Target to build") + parser.add_argument( + "--extra-cmake-defines", + type=str, + default="{}", + help=("JSON object of additional cmake defines"), + ) args, unknown_args = parser.parse_known_args() - return args.target, unknown_args + extra_defines = json.loads(args.extra_cmake_defines) + if not isinstance(extra_defines, dict): + print("--extra-cmake-defines must be a JSON object", file=sys.stderr) + sys.exit(1) + + return args.target, extra_defines, unknown_args def main() -> None: - target, command_line_args = get_command_line_args() + target, extra_cmake_defines, command_line_args = get_command_line_args() run_path = os.getcwd() parents = Path(__file__).parents @@ -40,14 +51,14 @@ def main() -> None: "Please run the script from the root of the FBOSS repository", file=sys.stderr, ) - exit(1) + sys.exit(1) expected_path = parents[3].absolute().as_posix() if run_path != expected_path: error_string = f"""Please executed the script from the root of the FBOSS repository Expected run path: {expected_path} Current run path: {run_path}""" print(error_string, file=sys.stderr) - exit(1) + sys.exit(1) get_deps_paths = [ expected_path + "/opensource/fbcode_builder/getdeps.py", @@ -60,7 +71,7 @@ def main() -> None: get_deps_path = maybe_path if get_deps_path is None: print("Could not find getdeps.py", file=sys.stderr) - exit(1) + sys.exit(1) else: print(get_deps_path) @@ -70,11 +81,16 @@ def main() -> None: if is_facebook_machine: get_deps_path = f"{proxy_env_vars} {get_deps_path}" + cmake_defines = {"CMAKE_BUILD_TYPE": "MinSizeRel", "CMAKE_CXX_STANDARD": "20"} + cmake_defines.update(extra_cmake_defines) + cmake_defines_json = json.dumps(cmake_defines) + print(f"Starting build for {target}") subprocess.run( f"""{get_deps_path} build """ - + '--allow-system-packages --num-jobs 32 --extra-cmake-defines=\'{"CMAKE_BUILD_TYPE": "MinSizeRel", "CMAKE_CXX_STANDARD": "20"}\' --cmake-target' + + f"--allow-system-packages --num-jobs 32 --extra-cmake-defines='{cmake_defines_json}' --src-dir {expected_path} --cmake-target" + f" {target} fboss", + check=False, shell=True, ) @@ -82,29 +98,32 @@ def main() -> None: show_build_dir_proc = subprocess.run( f"""{get_deps_path} show-build-dir fboss""", + check=False, shell=True, capture_output=True, text=True, ) - fboss_oss_target = ( - show_build_dir_proc.stdout.rstrip() - + f"/{target}" - + " " - + " ".join(command_line_args) - ) + build_dir = show_build_dir_proc.stdout.rstrip() + target_basename = target.removesuffix(".GEN_PY_EXE") + output_path = f"{build_dir}/{target_basename}" + + # Thrift python output is a directory; everything else is a native executable + python_prefix = "python3 " if os.path.isdir(output_path) else "" + fboss_oss_target = f"{python_prefix}{output_path} {' '.join(command_line_args)}" result = subprocess.run( fboss_oss_target, + check=False, shell=True, ) if result.returncode != 0: print(f"Failed to run target {target}", file=sys.stderr) - exit(1) + sys.exit(1) print("Configs have been written to specified output directory") - exit(0) + sys.exit(0) if __name__ == "__main__":