Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions python/private/pypi/extension.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
# Keeps track of all the hub's whl repos across the different versions.
# dict[hub, dict[whl, dict[version, str pip]]]
# Where hub, whl, and pip are the repo names
dep_graph = {}
hub_whl_map = {}
hub_group_map = {}
exposed_packages = {}
Expand All @@ -468,6 +469,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
else:
whl_libraries[whl_name] = lib

dep_graph[hub.name] = out.dep_graph
exposed_packages[hub.name] = out.exposed_packages
extra_aliases[hub.name] = out.extra_aliases
hub_group_map[hub.name] = out.group_map
Expand All @@ -477,6 +479,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
config = config,
declared_deps = declared_deps,
default_hub = config.default_hub or renamed_default_hub,
dep_graph = dep_graph,
exposed_packages = exposed_packages,
extra_aliases = extra_aliases,
facts = simpleapi_cache.get_facts(),
Expand Down Expand Up @@ -618,6 +621,7 @@ def _pip_impl(module_ctx):
hub_repository(
name = hub_name,
repo_name = hub_name,
dep_graph = mods.dep_graph.get(hub_name, {}),
extra_hub_aliases = mods.extra_aliases.get(hub_name, {}),
whl_map = {
key: whl_config_settings_to_json(values)
Expand Down
19 changes: 15 additions & 4 deletions python/private/pypi/hub_builder.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -118,15 +118,21 @@ def _build(self):
if self._logger.failed():
return ret

dep_graph = {}
whl_map = {}
for key, settings in self._whl_map.items():
for setting, repo in settings.items():
whl_map.setdefault(key, {}).setdefault(repo, []).append(setting)
whl_map.setdefault(key, {}).setdefault(repo.name, []).append(setting)
if repo.dependencies:
# TODO @aignas 2026-08-16: add the version to the dep_graph key
dep_graph[key] = repo.dependencies

return struct(
# The config settings for matching repo spokes.
# dict[str repo_name, dict[str repo_name, list[str]]]
whl_map = whl_map,
# The dependency_graph
dep_graph = dep_graph,
# Maps a wheel to a list of groups
# dict[str group_name, list[str]]
group_map = self._group_map,
Expand Down Expand Up @@ -295,14 +301,15 @@ def _diff_dict(first, second):
else:
return None

def _add_whl_library(self, *, python_version, whl, repo):
def _add_whl_library(self, *, python_version, whl, repo, dependencies):
"""Add a whl_library and kwargs to call it with for the hub.

Args:
self: implicitly added
python_version: {type}`str` the python version to assume
whl: struct from `_whl_library_args()`
repo: struct from `_whl_repo`
dependencies: the list of dependencies that this whl depends on
"""
if repo == None:
# NOTE @aignas 2025-07-07: we guard against an edge-case where there
Expand Down Expand Up @@ -335,7 +342,7 @@ def _add_whl_library(self, *, python_version, whl, repo):
self._whl_libraries[repo_name] = repo.args

mapping = self._whl_map.setdefault(whl.name, {})
if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name:
if repo.config_setting in mapping and mapping[repo.config_setting].name != repo_name:
fail(
"attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format(
mapping[repo.config_setting],
Expand All @@ -344,7 +351,10 @@ def _add_whl_library(self, *, python_version, whl, repo):
),
)
else:
mapping[repo.config_setting] = repo_name
mapping[repo.config_setting] = struct(
name = repo_name,
dependencies = dependencies,
)

### end of setters, below we have various functions to implement the public methods

Expand Down Expand Up @@ -560,6 +570,7 @@ def _create_whl_repos(
python_version = python_version,
whl = whl,
repo = repo,
dependencies = src.dependencies,
)

def _common_args(self, module_ctx, *, pip_attr):
Expand Down
9 changes: 9 additions & 0 deletions python/private/pypi/hub_repository.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def _impl(rctx):
},
extra_hub_aliases = rctx.attr.extra_hub_aliases,
requirement_cycles = rctx.attr.groups,
dep_graph = rctx.attr.dep_graph,
platform_config_settings = rctx.attr.platform_config_settings,
)
for path, contents in aliases.items():
Expand Down Expand Up @@ -71,6 +72,14 @@ def _impl(rctx):

hub_repository = repository_rule(
attrs = {
"dep_graph": attr.string_list_dict(
doc = """
The dependency graph that is a "{package} {version}" to list of Requires-Dist values.

This is so that we can handle different graphs.
""",
mandatory = False,
),
"extra_hub_aliases": attr.string_list_dict(
doc = "Extra aliases to make for specific wheels in the hub repo.",
mandatory = True,
Expand Down
9 changes: 9 additions & 0 deletions python/private/pypi/parse_requirements.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,13 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p
name = pkg["name"]
version = pkg["version"]
norm_name = normalize_name(name)
dependencies = pkg.get("dependencies", [])
if dependencies:
dependencies = [
"{name}; {marker}".format(**dep) if "marker" in dep else dep["name"]
for dep in dependencies
]

entry = uv_packages.setdefault(norm_name, {
"distribution": name,
"resolved_srcs": [],
Expand Down Expand Up @@ -278,6 +285,7 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p
)
entry["resolved_srcs"].append(struct(
distribution = name,
dependencies = dependencies,
extra_pip_args = extra_pip_args or [],
requirement_line = requirement_line,
target_platforms = plats,
Expand Down Expand Up @@ -520,6 +528,7 @@ def _package_srcs(
key,
struct(
distribution = name,
dependencies = [],
extra_pip_args = r.extra_pip_args,
requirement_line = req_line,
target_platforms = [],
Expand Down
52 changes: 49 additions & 3 deletions python/private/pypi/pkg_aliases.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,16 @@ setting maps to and their precedence, refer to documentation on that page.
load("@bazel_skylib//lib:selects.bzl", "selects")
load("//python/private:common_labels.bzl", "labels")
load("//python/private:text_util.bzl", "render")
load("//python/private/pypi:whl_library_targets.bzl", "whl_library_deps_targets")
load(
":labels.bzl",
"DATA_LABEL",
"DIST_INFO_LABEL",
"EXTRACTED_WHEEL_FILES",
"PY_LIBRARY_IMPL_LABEL",
"PY_LIBRARY_PUBLIC_LABEL",
"PY_SRCS_LABEL",
"WHEEL_FILE",
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
Expand Down Expand Up @@ -79,6 +82,10 @@ def pkg_aliases(
actual,
group_name = None,
extra_aliases = None,
requires_dist = [], # comes from METADATA or lock file
extras = [], # comes from METADATA or lock file
include = [], # comes from `//:config.bzl#packages`
group_deps = [], # comes as an arg
**kwargs):
"""Create aliases for an actual package.

Expand All @@ -91,9 +98,16 @@ def pkg_aliases(
the aliases to point to mapping to repositories. The keys are passed
to bazel skylib's `selects.with_or`, so they can be tuples as well.
group_name: {type}`str` The group name that the pkg belongs to.
group_deps: {type}`list[str]` The packages that are in the given group.
extra_aliases: {type}`list[str]` The extra aliases to be created.
requires_dist: {type}`list[str]` The list of dependencies.
extras: {type}`list[str]` The extras for which we should add extra
dependencies when parsing the requires_dist.
include: {type}`list[str]` The subset of packages to include.
**kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`.
"""
metadata_name = name

alias = kwargs.pop("native", native).alias
select = kwargs.pop("select", selects.with_or)

Expand All @@ -102,10 +116,40 @@ def pkg_aliases(
actual = ":" + PY_LIBRARY_PUBLIC_LABEL,
)

target_names = {
PY_LIBRARY_PUBLIC_LABEL: PY_LIBRARY_IMPL_LABEL if group_name else PY_LIBRARY_PUBLIC_LABEL,
WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if group_name else WHEEL_FILE_PUBLIC_LABEL,
if requires_dist:
# if it has a group_name set, then it will create the impl label target in the
# current macro, we still need to do the aliases to the actual groups package as
# per below
whl_library_deps_targets(
repo = None,
aliases = {}, # not none
metadata_name = metadata_name,
requires_dist = requires_dist,
extras = extras,
include = include,
group_deps = group_deps,
group_name = group_name,
dep_template = "//{name}:{target}", # this is const in this setting
visibility = ["//visibility:public"],
)
target_names = {}
else:
if group_name:
py_library_target = PY_LIBRARY_IMPL_LABEL
whl_target = WHEEL_FILE_IMPL_LABEL
else:
py_library_target = PY_LIBRARY_PUBLIC_LABEL
whl_target = WHEEL_FILE_PUBLIC_LABEL

target_names = {
PY_LIBRARY_PUBLIC_LABEL: py_library_target,
WHEEL_FILE_PUBLIC_LABEL: whl_target,
}

target_names = target_names | {
DATA_LABEL: DATA_LABEL,
WHEEL_FILE: WHEEL_FILE,
PY_SRCS_LABEL: PY_SRCS_LABEL,
DIST_INFO_LABEL: DIST_INFO_LABEL,
EXTRACTED_WHEEL_FILES: EXTRACTED_WHEEL_FILES,
} | {
Expand Down Expand Up @@ -157,6 +201,8 @@ def pkg_aliases(
kwargs = {}
if target_name.startswith("_"):
kwargs["visibility"] = ["//_groups:__subpackages__"]
elif target_name in [WHEEL_FILE, PY_SRCS_LABEL]:
kwargs["visibility"] = ["//visibility:private"]

alias(
name = target_name,
Expand Down
41 changes: 30 additions & 11 deletions python/private/pypi/render_pkg_aliases.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ load(
":generate_group_library_build_bazel.bzl",
"generate_group_library_build_bazel",
) # buildifier: disable=bzl-visibility
load(":pep508_requirement.bzl", "requirement")

NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\
No matching wheel for current configuration's Python version.
Expand Down Expand Up @@ -70,24 +71,22 @@ def _render_common_aliases(*, name, aliases, **kwargs):
"pkg_aliases",
name = repr(name),
actual = _repr_actual(aliases),
include = "packages",
**_repr_dict(**kwargs)
)
extra_loads = ""
if "whl_config_setting" in pkg_aliases:
extra_loads = """load("@rules_python//python/private/pypi:whl_config_setting.bzl", "whl_config_setting")"""
extra_loads += "\n"

return """\
load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases")
{extra_loads}
load("@rules_python//python/private/pypi:whl_config_setting.bzl", "whl_config_setting")
load("//:config.bzl", "packages")

package(default_visibility = ["//visibility:public"])

{aliases}""".format(
aliases = pkg_aliases,
extra_loads = extra_loads,
)

def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs):
def render_pkg_aliases(*, aliases, dep_graph = None, requirement_cycles = None, extra_hub_aliases = {}, **kwargs):
"""Create alias declarations for each PyPI package.

The aliases should be appended to the pip_repository BUILD.bazel file. These aliases
Expand All @@ -100,6 +99,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases
requirement_cycles: any package groups to also add.
extra_hub_aliases: The list of extra aliases for each whl to be added
in addition to the default ones.
dep_graph: The dep graph for the given wheels.
**kwargs: Extra kwargs to pass to the rules.

Returns:
Expand All @@ -111,6 +111,21 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases
elif type(aliases) != type({}):
fail("The aliases need to be provided as a dict, got: {}".format(type(aliases)))

aliases = {
normalize_name(name): pkg_aliases
for name, pkg_aliases in aliases.items()
}
dep_graph = dep_graph or {}
extras_per_package = {}
for _, dependencies in dep_graph.items():
for dep in dependencies:
dep = requirement(dep)
name = normalize_name(dep.name)
extras_per_package.setdefault(name, {}).update({
x: None
for x in dep.extras
})

whl_group_mapping = {}
if requirement_cycles:
requirement_cycles = {
Expand All @@ -125,18 +140,22 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases
}

files = {
"{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases(
name = normalize_name(name),
"{}/BUILD.bazel".format(name): _render_common_aliases(
name = name,
aliases = pkg_aliases,
extra_aliases = extra_hub_aliases.get(normalize_name(name), []),
group_name = whl_group_mapping.get(normalize_name(name)),
extra_aliases = extra_hub_aliases.get(name, []),
group_name = whl_group_mapping.get(name),
group_deps = requirement_cycles.get(name),
requires_dist = dep_graph.get(name, []),
extras = sorted(extras_per_package.get(name, [])),
**kwargs
).strip()
for name, pkg_aliases in aliases.items()
}

if requirement_cycles:
files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles)

return files

def _major_minor(python_version):
Expand Down
5 changes: 0 additions & 5 deletions python/private/pypi/whl_library_targets.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -373,13 +373,8 @@ def whl_library_deps_targets(
visibility = ["//visibility:public"],
native = native,
rules = struct(
copy_file = copy_file,
py_binary = py_binary,
py_library = py_library,
venv_entry_point = venv_entry_point,
venv_rewrite_shebang = venv_rewrite_shebang,
env_marker_setting = env_marker_setting,
create_inits = _create_inits,
)):
"""Create all of the whl_library targets.

Expand Down