From 2c843539e08943e357a0039293ed8267d836e685 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 14:22:33 +0900
Subject: [PATCH 01/13] wip
---
python/private/pypi/parse_requirements.bzl | 74 +++++++++++++---------
1 file changed, 43 insertions(+), 31 deletions(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index acf3b0c6ae..919779284f 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -88,6 +88,7 @@ def parse_requirements(
evaluate_markers = evaluate_markers or (lambda _ctx, _requirements: {})
options = {}
requirements = {}
+ reqs_with_env_markers = {}
for file, plats in requirements_by_platform.items():
logger.trace(lambda: "Using {} for {}".format(file, plats))
contents = ctx.read(file)
@@ -96,6 +97,37 @@ def parse_requirements(
# needed for the whl_library declarations later.
parse_result = parse_requirements_txt(contents)
+ tokenized_options = []
+ for opt in parse_result.options:
+ for p in opt.split(" "):
+ tokenized_options.append(p)
+
+ pip_args = tokenized_options + extra_pip_args
+ for plat in plats:
+ requirements[plat] = parse_result.requirements
+ for entry in parse_result.requirements:
+ requirement_line = entry[1]
+
+ # output all of the requirement lines that have a marker
+ if ";" in requirement_line:
+ reqs_with_env_markers.setdefault(requirement_line, []).append(plat)
+ options[plat] = pip_args
+
+ # This may call to Python, so execute it early (before calling to the
+ # internet below) and ensure that we call it only once.
+ #
+ # NOTE @aignas 2024-07-13: in the future, if this is something that we want
+ # to do, we could use Python to parse the requirement lines and infer the
+ # URL of the files to download things from. This should be important for
+ # VCS package references.
+ env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
+ logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
+ reqs_with_env_markers,
+ env_marker_target_platforms,
+ ))
+
+ requirements_by_platform = {}
+ for target_platform, reqs_ in requirements.items():
# Replicate a surprising behavior that WORKSPACE builds allowed:
# Defining a repo with the same name multiple times, but only the last
# definition is respected.
@@ -105,7 +137,7 @@ def parse_requirements(
# Lines with different markers are not condidered duplicates.
requirements_dict = {}
for entry in sorted(
- parse_result.requirements,
+ reqs_,
# Get the longest match and fallback to original WORKSPACE sorting,
# which should get us the entry with most extras.
#
@@ -114,33 +146,26 @@ def parse_requirements(
# should do this now.
key = lambda x: (len(x[1].partition("==")[0]), x),
):
- req = requirement(entry[1])
- requirements_dict[(req.name, req.version, req.marker)] = entry
+ req_line = entry[1]
+ req = requirement(req_line)
- tokenized_options = []
- for opt in parse_result.options:
- for p in opt.split(" "):
- tokenized_options.append(p)
+ if req.marker:
+ plats = env_marker_target_platforms.get(req_line, [])
+ if not plats:
+ continue
+ elif target_platform not in plats:
+ continue
- pip_args = tokenized_options + extra_pip_args
- for plat in plats:
- requirements[plat] = requirements_dict.values()
- options[plat] = pip_args
+ requirements_dict[req.name] = entry
- requirements_by_platform = {}
- reqs_with_env_markers = {}
- for target_platform, reqs_ in requirements.items():
extra_pip_args = options[target_platform]
- for distribution, requirement_line in reqs_:
+ for distribution, requirement_line in requirements_dict.values():
for_whl = requirements_by_platform.setdefault(
normalize_name(distribution),
{},
)
- if ";" in requirement_line:
- reqs_with_env_markers.setdefault(requirement_line, []).append(target_platform)
-
for_req = for_whl.setdefault(
(requirement_line, ",".join(extra_pip_args)),
struct(
@@ -153,19 +178,6 @@ def parse_requirements(
)
for_req.target_platforms.append(target_platform)
- # This may call to Python, so execute it early (before calling to the
- # internet below) and ensure that we call it only once.
- #
- # NOTE @aignas 2024-07-13: in the future, if this is something that we want
- # to do, we could use Python to parse the requirement lines and infer the
- # URL of the files to download things from. This should be important for
- # VCS package references.
- env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
- logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
- reqs_with_env_markers,
- env_marker_target_platforms,
- ))
-
index_urls = {}
if get_index_urls:
index_urls = get_index_urls(
From ede3fa1b11e009ef490c86c2e310d960132ae727 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 14:23:51 +0900
Subject: [PATCH 02/13] bring back the version
---
python/private/pypi/parse_requirements.bzl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 919779284f..aadadde5d5 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -156,7 +156,7 @@ def parse_requirements(
elif target_platform not in plats:
continue
- requirements_dict[req.name] = entry
+ requirements_dict[(req.name, req.version)] = entry
extra_pip_args = options[target_platform]
From 78e08b533e795cc97ee7e51f3eee14c88d9ba8c1 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 14:24:49 +0900
Subject: [PATCH 03/13] fix the assertion
---
tests/pypi/parse_requirements/parse_requirements_tests.bzl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index bd0078bfa4..54362efd63 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -766,7 +766,7 @@ def _test_get_index_urls_single_py_version(env):
env.expect.that_collection(got).contains_exactly([
struct(
is_exposed = True,
- is_multiple_versions = True,
+ is_multiple_versions = False,
name = "foo",
srcs = [
struct(
From 2a0a07630655337089d39727de07468b22f79f21 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 14:26:09 +0900
Subject: [PATCH 04/13] add a note
---
python/private/pypi/parse_requirements.bzl | 2 ++
1 file changed, 2 insertions(+)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index aadadde5d5..7a7eaa012e 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -156,6 +156,8 @@ def parse_requirements(
elif target_platform not in plats:
continue
+ # FIXME @aignas 2025-11-01: I don't think the `req.version` should be here
+ # because WORKSPACE would create files based on the `req.name`.
requirements_dict[(req.name, req.version)] = entry
extra_pip_args = options[target_platform]
From 25d1a0de8781deb2c8e896a7a8ab811939f021c0 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 14:27:12 +0900
Subject: [PATCH 05/13] typo
---
python/private/pypi/parse_requirements.bzl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 7a7eaa012e..7b289d0be7 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -134,7 +134,7 @@ def parse_requirements(
# The requirement lines might have duplicate names because lines for extras
# are returned as just the base package name. e.g., `foo[bar]` results
# in an entry like `("foo", "foo[bar] == 1.0 ...")`.
- # Lines with different markers are not condidered duplicates.
+ # Lines with different markers are not considered duplicates.
requirements_dict = {}
for entry in sorted(
reqs_,
From fd6eb82ef34bba32cb86ef1f4b24e677dc299a49 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 14:30:34 +0900
Subject: [PATCH 06/13] remove unnecessary code to show the simplification
---
python/private/pypi/parse_requirements.bzl | 11 ++---------
1 file changed, 2 insertions(+), 9 deletions(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 7b289d0be7..7a0e8a11bf 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -197,8 +197,7 @@ def parse_requirements(
for name, reqs in sorted(requirements_by_platform.items()):
requirement_target_platforms = {}
for r in reqs.values():
- target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms)
- for p in target_platforms:
+ for p in r.target_platforms:
requirement_target_platforms[p] = None
item = struct(
@@ -211,7 +210,6 @@ def parse_requirements(
reqs = reqs,
index_urls = index_urls,
platforms = platforms,
- env_marker_target_platforms = env_marker_target_platforms,
extract_url_srcs = extract_url_srcs,
logger = logger,
),
@@ -235,18 +233,13 @@ def _package_srcs(
index_urls,
platforms,
logger,
- env_marker_target_platforms,
extract_url_srcs):
"""A function to return sources for a particular package."""
srcs = {}
for r in sorted(reqs.values(), key = lambda r: r.requirement_line):
- if ";" in r.requirement_line:
- target_platforms = env_marker_target_platforms.get(r.requirement_line, [])
- else:
- target_platforms = r.target_platforms
extra_pip_args = tuple(r.extra_pip_args)
- for target_platform in target_platforms:
+ for target_platform in r.target_platforms:
if platforms and target_platform not in platforms:
fail("The target platform '{}' could not be found in {}".format(
target_platform,
From b85d57f047c5d04a4aee1adecc938843a80fcda3 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 22:58:46 +0900
Subject: [PATCH 07/13] improve the tests and change the grouping of the
requirements
---
python/private/pypi/parse_requirements.bzl | 4 +-
.../parse_requirements_tests.bzl | 40 ++++++++++++-------
2 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 7a0e8a11bf..0eb91a6014 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -156,9 +156,7 @@ def parse_requirements(
elif target_platform not in plats:
continue
- # FIXME @aignas 2025-11-01: I don't think the `req.version` should be here
- # because WORKSPACE would create files based on the `req.name`.
- requirements_dict[(req.name, req.version)] = entry
+ requirements_dict[req.name] = entry
extra_pip_args = options[target_platform]
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index 54362efd63..78cc62d05a 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -22,12 +22,6 @@ load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier:
def _mock_ctx():
testdata = {
- "requirements_different_package_version": """\
-foo==0.0.1+local \
- --hash=sha256:deadbeef
-foo==0.0.1 \
- --hash=sha256:deadb00f
-""",
"requirements_direct": """\
foo[extra] @ https://some-url/package.whl
""",
@@ -39,6 +33,14 @@ foo @ https://github.com/org/foo/downloads/foo-1.1.tar.gz
foo[extra]==0.0.1 \
--hash=sha256:deadbeef
+""",
+ "requirements_foo": """\
+foo==0.0.1 \
+ --hash=sha256:deadb00f
+""",
+ "requirements_foo_local": """\
+foo==0.0.1+local \
+ --hash=sha256:deadbeef
""",
"requirements_git": """
foo @ git+https://github.com/org/foo.git@deadbeef
@@ -75,7 +77,7 @@ foo==0.0.2; python_full_version >= '3.10.0' \
--hash=sha256:deadb11f
""",
"requirements_optional_hash": """
-foo==0.0.4 @ https://example.org/foo-0.0.4.whl
+bar==0.0.4 @ https://example.org/bar-0.0.4.whl
foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef
""",
"requirements_osx": """\
@@ -441,7 +443,8 @@ _tests.append(_test_env_marker_resolution)
def _test_different_package_version(env):
got = parse_requirements(
requirements_by_platform = {
- "requirements_different_package_version": ["linux_x86_64"],
+ "requirements_foo": ["linux_aarch64"],
+ "requirements_foo_local": ["linux_x86_64"],
},
)
env.expect.that_collection(got).contains_exactly([
@@ -454,7 +457,7 @@ def _test_different_package_version(env):
distribution = "foo",
extra_pip_args = [],
requirement_line = "foo==0.0.1 --hash=sha256:deadb00f",
- target_platforms = ["linux_x86_64"],
+ target_platforms = ["linux_aarch64"],
url = "",
filename = "",
sha256 = "",
@@ -484,20 +487,27 @@ def _test_optional_hash(env):
)
env.expect.that_collection(got).contains_exactly([
struct(
- name = "foo",
+ name = "bar",
is_exposed = True,
- is_multiple_versions = True,
+ is_multiple_versions = False,
srcs = [
struct(
- distribution = "foo",
+ distribution = "bar",
extra_pip_args = [],
- requirement_line = "foo==0.0.4",
+ requirement_line = "bar==0.0.4",
target_platforms = ["linux_x86_64"],
- url = "https://example.org/foo-0.0.4.whl",
- filename = "foo-0.0.4.whl",
+ url = "https://example.org/bar-0.0.4.whl",
+ filename = "bar-0.0.4.whl",
sha256 = "",
yanked = False,
),
+ ],
+ ),
+ struct(
+ name = "foo",
+ is_exposed = True,
+ is_multiple_versions = False,
+ srcs = [
struct(
distribution = "foo",
extra_pip_args = [],
From 1cc6484898ecdea41a642290a6c97a3ac62e197f Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 22:59:34 +0900
Subject: [PATCH 08/13] reword
---
python/private/pypi/parse_requirements.bzl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 0eb91a6014..a0ea305652 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -127,7 +127,7 @@ def parse_requirements(
))
requirements_by_platform = {}
- for target_platform, reqs_ in requirements.items():
+ for target_platform, parse_results in requirements.items():
# Replicate a surprising behavior that WORKSPACE builds allowed:
# Defining a repo with the same name multiple times, but only the last
# definition is respected.
@@ -137,7 +137,7 @@ def parse_requirements(
# Lines with different markers are not considered duplicates.
requirements_dict = {}
for entry in sorted(
- reqs_,
+ parse_results,
# Get the longest match and fallback to original WORKSPACE sorting,
# which should get us the entry with most extras.
#
From f6cc2826c50add3ad73e0bae51d55522929e16a1 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 23:03:52 +0900
Subject: [PATCH 09/13] remove an outdated comment
---
python/private/pypi/parse_requirements.bzl | 1 -
1 file changed, 1 deletion(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index a0ea305652..c3c5b31546 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -134,7 +134,6 @@ def parse_requirements(
# The requirement lines might have duplicate names because lines for extras
# are returned as just the base package name. e.g., `foo[bar]` results
# in an entry like `("foo", "foo[bar] == 1.0 ...")`.
- # Lines with different markers are not considered duplicates.
requirements_dict = {}
for entry in sorted(
parse_results,
From 60fb1e9707d439bad5fca73b5929631478f4285d Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 1 Nov 2025 23:04:55 +0900
Subject: [PATCH 10/13] remove another comment as pipstar is the direction we
are moving in
---
python/private/pypi/parse_requirements.bzl | 5 -----
1 file changed, 5 deletions(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index c3c5b31546..5ec6ee93bb 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -115,11 +115,6 @@ def parse_requirements(
# This may call to Python, so execute it early (before calling to the
# internet below) and ensure that we call it only once.
- #
- # NOTE @aignas 2024-07-13: in the future, if this is something that we want
- # to do, we could use Python to parse the requirement lines and infer the
- # URL of the files to download things from. This should be important for
- # VCS package references.
env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
reqs_with_env_markers,
From 4bc73f34b126b7e712ae63036de240209abd95bf Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sun, 2 Nov 2025 00:01:34 +0900
Subject: [PATCH 11/13] refactor logging and naming
---
python/private/pypi/parse_requirements.bzl | 18 +++++++-----------
.../parse_requirements_tests.bzl | 2 --
2 files changed, 7 insertions(+), 13 deletions(-)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 5ec6ee93bb..7d210abbaa 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -115,14 +115,14 @@ def parse_requirements(
# This may call to Python, so execute it early (before calling to the
# internet below) and ensure that we call it only once.
- env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
+ resolved_marker_platforms = evaluate_markers(ctx, reqs_with_env_markers)
logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
reqs_with_env_markers,
- env_marker_target_platforms,
+ resolved_marker_platforms,
))
requirements_by_platform = {}
- for target_platform, parse_results in requirements.items():
+ for plat, parse_results in requirements.items():
# Replicate a surprising behavior that WORKSPACE builds allowed:
# Defining a repo with the same name multiple times, but only the last
# definition is respected.
@@ -143,16 +143,12 @@ def parse_requirements(
req_line = entry[1]
req = requirement(req_line)
- if req.marker:
- plats = env_marker_target_platforms.get(req_line, [])
- if not plats:
- continue
- elif target_platform not in plats:
- continue
+ if req.marker and plat not in resolved_marker_platforms.get(req_line, []):
+ continue
requirements_dict[req.name] = entry
- extra_pip_args = options[target_platform]
+ extra_pip_args = options[plat]
for distribution, requirement_line in requirements_dict.values():
for_whl = requirements_by_platform.setdefault(
@@ -170,7 +166,7 @@ def parse_requirements(
extra_pip_args = extra_pip_args,
),
)
- for_req.target_platforms.append(target_platform)
+ for_req.target_platforms.append(plat)
index_urls = {}
if get_index_urls:
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index 78cc62d05a..1e57dfb1e4 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -696,7 +696,6 @@ def _test_get_index_urls_different_versions(env):
),
},
),
- debug = True,
)
env.expect.that_collection(got).contains_exactly([
@@ -770,7 +769,6 @@ def _test_get_index_urls_single_py_version(env):
),
},
),
- debug = True,
)
env.expect.that_collection(got).contains_exactly([
From aa5d088f900cbd0bcbcf0411dc4d4c555ae23dd3 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sun, 2 Nov 2025 00:37:59 +0900
Subject: [PATCH 12/13] add a test
---
.../parse_requirements_tests.bzl | 39 +++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index 1e57dfb1e4..63755d2edd 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -479,6 +479,45 @@ def _test_different_package_version(env):
_tests.append(_test_different_package_version)
+def _test_different_package_extras(env):
+ got = parse_requirements(
+ requirements_by_platform = {
+ "requirements_foo": ["linux_aarch64"],
+ "requirements_lock": ["linux_x86_64"],
+ },
+ )
+ env.expect.that_collection(got).contains_exactly([
+ struct(
+ name = "foo",
+ is_exposed = True,
+ is_multiple_versions = True,
+ srcs = [
+ struct(
+ distribution = "foo",
+ extra_pip_args = [],
+ requirement_line = "foo==0.0.1 --hash=sha256:deadb00f",
+ target_platforms = ["linux_aarch64"],
+ url = "",
+ filename = "",
+ sha256 = "",
+ yanked = False,
+ ),
+ struct(
+ distribution = "foo",
+ extra_pip_args = [],
+ requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
+ target_platforms = ["linux_x86_64"],
+ url = "",
+ filename = "",
+ sha256 = "",
+ yanked = False,
+ ),
+ ],
+ ),
+ ])
+
+_tests.append(_test_different_package_extras)
+
def _test_optional_hash(env):
got = parse_requirements(
requirements_by_platform = {
From f6932393bd31ab1344f8d9cb95671ca44c0e30fe Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sun, 2 Nov 2025 00:59:52 +0900
Subject: [PATCH 13/13] fix #3374
---
python/private/pypi/hub_builder.bzl | 7 +-
python/private/pypi/whl_repo_name.bzl | 6 +-
tests/pypi/hub_builder/hub_builder_tests.bzl | 160 +++++++++++++++++--
3 files changed, 159 insertions(+), 14 deletions(-)
diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl
index 58d35f2681..7cf60ff85f 100644
--- a/python/private/pypi/hub_builder.bzl
+++ b/python/private/pypi/hub_builder.bzl
@@ -566,8 +566,13 @@ def _whl_repo(
for p in src.target_platforms
]
+ # TODO @aignas 2025-11-02: once we have pipstar enabled we can add extra
+ # targets to each hub for each extra combination and solve this more cleanly as opposed to
+ # duplicating whl_library repositories.
+ target_platforms = src.target_platforms if is_multiple_versions else []
+
return struct(
- repo_name = whl_repo_name(src.filename, src.sha256),
+ repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms),
args = args,
config_setting = whl_config_setting(
version = python_version,
diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl
index 2b3b5418aa..29d774c361 100644
--- a/python/private/pypi/whl_repo_name.bzl
+++ b/python/private/pypi/whl_repo_name.bzl
@@ -18,12 +18,14 @@
load("//python/private:normalize_name.bzl", "normalize_name")
load(":parse_whl_name.bzl", "parse_whl_name")
-def whl_repo_name(filename, sha256):
+def whl_repo_name(filename, sha256, *target_platforms):
"""Return a valid whl_library repo name given a distribution filename.
Args:
filename: {type}`str` the filename of the distribution.
sha256: {type}`str` the sha256 of the distribution.
+ *target_platforms: {type}`list[str]` the extra suffixes to append.
+ Only used when we need to support different extras per version.
Returns:
a string that can be used in {obj}`whl_library`.
@@ -59,6 +61,8 @@ def whl_repo_name(filename, sha256):
elif version:
parts.insert(1, version)
+ parts.extend([p.partition("_")[-1] for p in target_platforms])
+
return "_".join(parts)
def pypi_repo_name(whl_name, *target_platforms):
diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl
index ee6200a70a..6d061f4d56 100644
--- a/tests/pypi/hub_builder/hub_builder_tests.bzl
+++ b/tests/pypi/hub_builder/hub_builder_tests.bzl
@@ -203,6 +203,142 @@ def _test_simple_multiple_requirements(env):
_tests.append(_test_simple_multiple_requirements)
+def _test_simple_extras_vs_no_extras(env):
+ builder = hub_builder(env)
+ builder.pip_parse(
+ _mock_mctx(
+ read = lambda x: {
+ "darwin.txt": "simple[foo]==0.0.1 --hash=sha256:deadbeef",
+ "win.txt": "simple==0.0.1 --hash=sha256:deadbeef",
+ }[x],
+ ),
+ _parse(
+ hub_name = "pypi",
+ python_version = "3.15",
+ requirements_darwin = "darwin.txt",
+ requirements_windows = "win.txt",
+ ),
+ )
+ pypi = builder.build()
+
+ pypi.exposed_packages().contains_exactly(["simple"])
+ pypi.group_map().contains_exactly({})
+ pypi.whl_map().contains_exactly({
+ "simple": {
+ "pypi_315_simple_osx_aarch64": [
+ whl_config_setting(
+ target_platforms = [
+ "cp315_osx_aarch64",
+ ],
+ version = "3.15",
+ ),
+ ],
+ "pypi_315_simple_windows_aarch64": [
+ whl_config_setting(
+ target_platforms = [
+ "cp315_windows_aarch64",
+ ],
+ version = "3.15",
+ ),
+ ],
+ },
+ })
+ pypi.whl_libraries().contains_exactly({
+ "pypi_315_simple_osx_aarch64": {
+ "config_load": "@pypi//:config.bzl",
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "simple[foo]==0.0.1 --hash=sha256:deadbeef",
+ },
+ "pypi_315_simple_windows_aarch64": {
+ "config_load": "@pypi//:config.bzl",
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "simple==0.0.1 --hash=sha256:deadbeef",
+ },
+ })
+ pypi.extra_aliases().contains_exactly({})
+
+_tests.append(_test_simple_extras_vs_no_extras)
+
+def _test_simple_extras_vs_no_extras_simpleapi(env):
+ def mocksimpleapi_download(*_, **__):
+ return {
+ "simple": parse_simpleapi_html(
+ url = "https://example.com",
+ content = """\
+ simple-0.0.1-py3-none-any.whl
+""",
+ ),
+ }
+
+ builder = hub_builder(
+ env,
+ simpleapi_download_fn = mocksimpleapi_download,
+ )
+ builder.pip_parse(
+ _mock_mctx(
+ read = lambda x: {
+ "darwin.txt": "simple[foo]==0.0.1 --hash=sha256:deadbeef",
+ "win.txt": "simple==0.0.1 --hash=sha256:deadbeef",
+ }[x],
+ ),
+ _parse(
+ hub_name = "pypi",
+ python_version = "3.15",
+ requirements_darwin = "darwin.txt",
+ requirements_windows = "win.txt",
+ experimental_index_url = "example.com",
+ ),
+ )
+ pypi = builder.build()
+
+ pypi.exposed_packages().contains_exactly(["simple"])
+ pypi.group_map().contains_exactly({})
+ pypi.whl_map().contains_exactly({
+ "simple": {
+ "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": [
+ whl_config_setting(
+ target_platforms = [
+ "cp315_osx_aarch64",
+ ],
+ version = "3.15",
+ ),
+ ],
+ "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": [
+ whl_config_setting(
+ target_platforms = [
+ "cp315_windows_aarch64",
+ ],
+ version = "3.15",
+ ),
+ ],
+ },
+ })
+ pypi.whl_libraries().contains_exactly({
+ "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": {
+ "config_load": "@pypi//:config.bzl",
+ "dep_template": "@pypi//{name}:{target}",
+ "filename": "simple-0.0.1-py3-none-any.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "simple[foo]==0.0.1",
+ "sha256": "deadbeef",
+ "urls": ["https://example.com/simple-0.0.1-py3-none-any.whl"],
+ },
+ "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": {
+ "config_load": "@pypi//:config.bzl",
+ "dep_template": "@pypi//{name}:{target}",
+ "filename": "simple-0.0.1-py3-none-any.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "simple==0.0.1",
+ "sha256": "deadbeef",
+ "urls": ["https://example.com/simple-0.0.1-py3-none-any.whl"],
+ },
+ })
+ pypi.extra_aliases().contains_exactly({})
+
+_tests.append(_test_simple_extras_vs_no_extras_simpleapi)
+
def _test_simple_multiple_python_versions(env):
builder = hub_builder(
env,
@@ -496,34 +632,34 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
pypi.group_map().contains_exactly({})
pypi.whl_map().contains_exactly({
"torch": {
- "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [
+ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef_linux_x86_64": [
whl_config_setting(
- target_platforms = ("cp312_linux_x86_64",),
+ target_platforms = ["cp312_linux_x86_64"],
version = "3.12",
),
],
- "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [
+ "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": [
whl_config_setting(
- target_platforms = ("cp312_linux_aarch64",),
+ target_platforms = ["cp312_linux_aarch64"],
version = "3.12",
),
],
- "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [
+ "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": [
whl_config_setting(
- target_platforms = ("cp312_windows_x86_64",),
+ target_platforms = ["cp312_windows_x86_64"],
version = "3.12",
),
],
- "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [
+ "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": [
whl_config_setting(
- target_platforms = ("cp312_osx_aarch64",),
+ target_platforms = ["cp312_osx_aarch64"],
version = "3.12",
),
],
},
})
pypi.whl_libraries().contains_exactly({
- "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": {
+ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef_linux_x86_64": {
"config_load": "@pypi//:config.bzl",
"dep_template": "@pypi//{name}:{target}",
"filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
@@ -532,7 +668,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
"sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364",
"urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"],
},
- "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": {
+ "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": {
"config_load": "@pypi//:config.bzl",
"dep_template": "@pypi//{name}:{target}",
"filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
@@ -541,7 +677,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
"sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a",
"urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"],
},
- "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": {
+ "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": {
"config_load": "@pypi//:config.bzl",
"dep_template": "@pypi//{name}:{target}",
"filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
@@ -550,7 +686,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
"sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97",
"urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"],
},
- "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": {
+ "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": {
"config_load": "@pypi//:config.bzl",
"dep_template": "@pypi//{name}:{target}",
"filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",