From 7cab002504b6537aa4426a125b10535882af7295 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 15 Aug 2026 09:55:45 +0900 Subject: [PATCH] feat: use uv.lock deps for the dep graph instead Summary: - Thread deps from uv.lock file into the hub_repository - Use them to define the dep graph - Alter the pkg_aliases and render_pkg_aliases to be able to use the dep graph information if it is provided. TODO: - [ ] Split the whl_library_targets into separate files, I think they have stabilized enough. - [ ] Have a separate PR just for threading the dependencies to render_pkg_aliases. We probably need to handle various edge-cases where we have different package versions in the uv.lock file. - [ ] Have a separate PR to enable usage of the dependency graph. This is now probably doing a few shortcuts. - [ ] If we have uv.lock, define fewer wheel repos - only define the sources without passing the dep_template in there. - [ ] Test coverage for the above. --- python/private/pypi/extension.bzl | 4 ++ python/private/pypi/hub_builder.bzl | 19 ++++++-- python/private/pypi/hub_repository.bzl | 9 ++++ python/private/pypi/parse_requirements.bzl | 9 ++++ python/private/pypi/pkg_aliases.bzl | 52 +++++++++++++++++++-- python/private/pypi/render_pkg_aliases.bzl | 41 +++++++++++----- python/private/pypi/whl_library_targets.bzl | 5 -- 7 files changed, 116 insertions(+), 23 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ed1fa28664..ab25936200 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -454,6 +454,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # Keeps track of all the hub's whl repos across the different versions. # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names + dep_graph = {} hub_whl_map = {} hub_group_map = {} exposed_packages = {} @@ -468,6 +469,7 @@ You cannot use both the additive_build_content and additive_build_content_file a else: whl_libraries[whl_name] = lib + dep_graph[hub.name] = out.dep_graph exposed_packages[hub.name] = out.exposed_packages extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map @@ -477,6 +479,7 @@ You cannot use both the additive_build_content and additive_build_content_file a config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, + dep_graph = dep_graph, exposed_packages = exposed_packages, extra_aliases = extra_aliases, facts = simpleapi_cache.get_facts(), @@ -618,6 +621,7 @@ def _pip_impl(module_ctx): hub_repository( name = hub_name, repo_name = hub_name, + dep_graph = mods.dep_graph.get(hub_name, {}), extra_hub_aliases = mods.extra_aliases.get(hub_name, {}), whl_map = { key: whl_config_settings_to_json(values) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 596cfb2cbe..c4f2a9058e 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -118,15 +118,21 @@ def _build(self): if self._logger.failed(): return ret + dep_graph = {} whl_map = {} for key, settings in self._whl_map.items(): for setting, repo in settings.items(): - whl_map.setdefault(key, {}).setdefault(repo, []).append(setting) + whl_map.setdefault(key, {}).setdefault(repo.name, []).append(setting) + if repo.dependencies: + # TODO @aignas 2026-08-16: add the version to the dep_graph key + dep_graph[key] = repo.dependencies return struct( # The config settings for matching repo spokes. # dict[str repo_name, dict[str repo_name, list[str]]] whl_map = whl_map, + # The dependency_graph + dep_graph = dep_graph, # Maps a wheel to a list of groups # dict[str group_name, list[str]] group_map = self._group_map, @@ -295,7 +301,7 @@ def _diff_dict(first, second): else: return None -def _add_whl_library(self, *, python_version, whl, repo): +def _add_whl_library(self, *, python_version, whl, repo, dependencies): """Add a whl_library and kwargs to call it with for the hub. Args: @@ -303,6 +309,7 @@ def _add_whl_library(self, *, python_version, whl, repo): python_version: {type}`str` the python version to assume whl: struct from `_whl_library_args()` repo: struct from `_whl_repo` + dependencies: the list of dependencies that this whl depends on """ if repo == None: # NOTE @aignas 2025-07-07: we guard against an edge-case where there @@ -335,7 +342,7 @@ def _add_whl_library(self, *, python_version, whl, repo): self._whl_libraries[repo_name] = repo.args mapping = self._whl_map.setdefault(whl.name, {}) - if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: + if repo.config_setting in mapping and mapping[repo.config_setting].name != repo_name: fail( "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( mapping[repo.config_setting], @@ -344,7 +351,10 @@ def _add_whl_library(self, *, python_version, whl, repo): ), ) else: - mapping[repo.config_setting] = repo_name + mapping[repo.config_setting] = struct( + name = repo_name, + dependencies = dependencies, + ) ### end of setters, below we have various functions to implement the public methods @@ -560,6 +570,7 @@ def _create_whl_repos( python_version = python_version, whl = whl, repo = repo, + dependencies = src.dependencies, ) def _common_args(self, module_ctx, *, pip_attr): diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..809df1ede1 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -34,6 +34,7 @@ def _impl(rctx): }, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, + dep_graph = rctx.attr.dep_graph, platform_config_settings = rctx.attr.platform_config_settings, ) for path, contents in aliases.items(): @@ -71,6 +72,14 @@ def _impl(rctx): hub_repository = repository_rule( attrs = { + "dep_graph": attr.string_list_dict( + doc = """ +The dependency graph that is a "{package} {version}" to list of Requires-Dist values. + +This is so that we can handle different graphs. +""", + mandatory = False, + ), "extra_hub_aliases": attr.string_list_dict( doc = "Extra aliases to make for specific wheels in the hub repo.", mandatory = True, diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 4874c7ac79..f784df8f85 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -172,6 +172,13 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p name = pkg["name"] version = pkg["version"] norm_name = normalize_name(name) + dependencies = pkg.get("dependencies", []) + if dependencies: + dependencies = [ + "{name}; {marker}".format(**dep) if "marker" in dep else dep["name"] + for dep in dependencies + ] + entry = uv_packages.setdefault(norm_name, { "distribution": name, "resolved_srcs": [], @@ -278,6 +285,7 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p ) entry["resolved_srcs"].append(struct( distribution = name, + dependencies = dependencies, extra_pip_args = extra_pip_args or [], requirement_line = requirement_line, target_platforms = plats, @@ -520,6 +528,7 @@ def _package_srcs( key, struct( distribution = name, + dependencies = [], extra_pip_args = r.extra_pip_args, requirement_line = req_line, target_platforms = [], diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index b1c29c95ee..d3a6540656 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -34,6 +34,7 @@ setting maps to and their precedence, refer to documentation on that page. load("@bazel_skylib//lib:selects.bzl", "selects") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") +load("//python/private/pypi:whl_library_targets.bzl", "whl_library_deps_targets") load( ":labels.bzl", "DATA_LABEL", @@ -41,6 +42,8 @@ load( "EXTRACTED_WHEEL_FILES", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", + "PY_SRCS_LABEL", + "WHEEL_FILE", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) @@ -79,6 +82,10 @@ def pkg_aliases( actual, group_name = None, extra_aliases = None, + requires_dist = [], # comes from METADATA or lock file + extras = [], # comes from METADATA or lock file + include = [], # comes from `//:config.bzl#packages` + group_deps = [], # comes as an arg **kwargs): """Create aliases for an actual package. @@ -91,9 +98,16 @@ def pkg_aliases( the aliases to point to mapping to repositories. The keys are passed to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. + group_deps: {type}`list[str]` The packages that are in the given group. extra_aliases: {type}`list[str]` The extra aliases to be created. + requires_dist: {type}`list[str]` The list of dependencies. + extras: {type}`list[str]` The extras for which we should add extra + dependencies when parsing the requires_dist. + include: {type}`list[str]` The subset of packages to include. **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ + metadata_name = name + alias = kwargs.pop("native", native).alias select = kwargs.pop("select", selects.with_or) @@ -102,10 +116,40 @@ def pkg_aliases( actual = ":" + PY_LIBRARY_PUBLIC_LABEL, ) - target_names = { - PY_LIBRARY_PUBLIC_LABEL: PY_LIBRARY_IMPL_LABEL if group_name else PY_LIBRARY_PUBLIC_LABEL, - WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if group_name else WHEEL_FILE_PUBLIC_LABEL, + if requires_dist: + # if it has a group_name set, then it will create the impl label target in the + # current macro, we still need to do the aliases to the actual groups package as + # per below + whl_library_deps_targets( + repo = None, + aliases = {}, # not none + metadata_name = metadata_name, + requires_dist = requires_dist, + extras = extras, + include = include, + group_deps = group_deps, + group_name = group_name, + dep_template = "//{name}:{target}", # this is const in this setting + visibility = ["//visibility:public"], + ) + target_names = {} + else: + if group_name: + py_library_target = PY_LIBRARY_IMPL_LABEL + whl_target = WHEEL_FILE_IMPL_LABEL + else: + py_library_target = PY_LIBRARY_PUBLIC_LABEL + whl_target = WHEEL_FILE_PUBLIC_LABEL + + target_names = { + PY_LIBRARY_PUBLIC_LABEL: py_library_target, + WHEEL_FILE_PUBLIC_LABEL: whl_target, + } + + target_names = target_names | { DATA_LABEL: DATA_LABEL, + WHEEL_FILE: WHEEL_FILE, + PY_SRCS_LABEL: PY_SRCS_LABEL, DIST_INFO_LABEL: DIST_INFO_LABEL, EXTRACTED_WHEEL_FILES: EXTRACTED_WHEEL_FILES, } | { @@ -157,6 +201,8 @@ def pkg_aliases( kwargs = {} if target_name.startswith("_"): kwargs["visibility"] = ["//_groups:__subpackages__"] + elif target_name in [WHEEL_FILE, PY_SRCS_LABEL]: + kwargs["visibility"] = ["//visibility:private"] alias( name = target_name, diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..0eda767492 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,6 +22,7 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility +load(":pep508_requirement.bzl", "requirement") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -70,24 +71,22 @@ def _render_common_aliases(*, name, aliases, **kwargs): "pkg_aliases", name = repr(name), actual = _repr_actual(aliases), + include = "packages", **_repr_dict(**kwargs) ) - extra_loads = "" - if "whl_config_setting" in pkg_aliases: - extra_loads = """load("@rules_python//python/private/pypi:whl_config_setting.bzl", "whl_config_setting")""" - extra_loads += "\n" return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") -{extra_loads} +load("@rules_python//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") +load("//:config.bzl", "packages") + package(default_visibility = ["//visibility:public"]) {aliases}""".format( aliases = pkg_aliases, - extra_loads = extra_loads, ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases(*, aliases, dep_graph = None, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +99,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + dep_graph: The dep graph for the given wheels. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -111,6 +111,21 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases elif type(aliases) != type({}): fail("The aliases need to be provided as a dict, got: {}".format(type(aliases))) + aliases = { + normalize_name(name): pkg_aliases + for name, pkg_aliases in aliases.items() + } + dep_graph = dep_graph or {} + extras_per_package = {} + for _, dependencies in dep_graph.items(): + for dep in dependencies: + dep = requirement(dep) + name = normalize_name(dep.name) + extras_per_package.setdefault(name, {}).update({ + x: None + for x in dep.extras + }) + whl_group_mapping = {} if requirement_cycles: requirement_cycles = { @@ -125,11 +140,14 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases } files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + "{}/BUILD.bazel".format(name): _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), + extra_aliases = extra_hub_aliases.get(name, []), + group_name = whl_group_mapping.get(name), + group_deps = requirement_cycles.get(name), + requires_dist = dep_graph.get(name, []), + extras = sorted(extras_per_package.get(name, [])), **kwargs ).strip() for name, pkg_aliases in aliases.items() @@ -137,6 +155,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) + return files def _major_minor(python_version): diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index b7fdbd55e9..afe8336868 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -373,13 +373,8 @@ def whl_library_deps_targets( visibility = ["//visibility:public"], native = native, rules = struct( - copy_file = copy_file, - py_binary = py_binary, py_library = py_library, - venv_entry_point = venv_entry_point, - venv_rewrite_shebang = venv_rewrite_shebang, env_marker_setting = env_marker_setting, - create_inits = _create_inits, )): """Create all of the whl_library targets.