Skip to content

Commit 0a77399

Browse files
nickatundonickatundo
committed
[#3] Download dev headers if not present
1 parent 0d4abb0 commit 0a77399

3 files changed

Lines changed: 227 additions & 36 deletions

File tree

‎python/setup.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
# setup.py
2+
import os
3+
24
from setuptools import Extension, setup
35

46
module = Extension(
@@ -10,6 +12,7 @@
1012
"src/ext/cJSON/cJSON.c",
1113
],
1214
extra_compile_args=["-O0", "-Isrc/ext/cJSON", "-std=c99"],
15+
include_dirs=os.environ["PYTHON_INCLUDE_DIRS"].split(os.pathsep),
1316
)
1417

1518
setup(

‎python/src/ubeacon/extension/download_python_headers.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
"""
2+
Download Python development headers for a given version into a temporary directory.
3+
4+
Works without root on Ubuntu, RHEL, Fedora, and OpenSUSE by downloading (not installing)
5+
the appropriate package and extracting it locally.
6+
"""
7+
8+
from __future__ import annotations
9+
10+
import subprocess
11+
from pathlib import Path
12+
13+
import distro
14+
15+
16+
def _run_command(command: list[str], **kwargs: object) -> subprocess.CompletedProcess:
17+
"""
18+
Run a command with subprocess.run and check for errors.
19+
20+
command: The command to run, as a list of arguments.
21+
kwargs: Additional keyword arguments to pass to subprocess.run.
22+
23+
Returns the CompletedProcess object returned by subprocess.run.
24+
"""
25+
return subprocess.run(command, check=True, capture_output=True, text=True, **kwargs)
26+
27+
28+
def _get_download_package_urls(version: str) -> str:
29+
"""
30+
Return the URL of a distribution package file.
31+
"""
32+
33+
match distro.id():
34+
case dist if dist in {"ubuntu", "debian"}:
35+
36+
return (
37+
_run_command(
38+
["apt-get", "--print-uris", "download", f"libpython{version}-dev"]
39+
)
40+
.stdout.splitlines()[0]
41+
.split()[0]
42+
.strip("'")
43+
)
44+
45+
case dist if dist in {"fedora", "rhel", "rocky", "centos", "amzn"}:
46+
47+
def dnf(package_name: str) -> str:
48+
# TODO support on ARM
49+
return _run_command(
50+
[
51+
"dnf",
52+
"repoquery",
53+
"--location",
54+
package_name,
55+
"--archlist",
56+
"x86_64",
57+
]
58+
).stdout.strip()
59+
60+
rpms = dnf(f"python{version}-devel")
61+
if not rpms:
62+
# The package may be named python3-devel if it's the default
63+
# Python version for the current distro version.
64+
rpms = dnf("python3-devel")
65+
assert version in rpms, f"Expected to find version {version} in package name: {rpms}"
66+
return rpms.splitlines()[
67+
0
68+
] # Take the first result if there are multiple matches.
69+
70+
case _:
71+
raise ValueError(f"Unsupported distribution: {distro.id()}")
72+
73+
74+
def _extract_deb(package_path: Path, extract_dir: Path) -> None:
75+
"""Extract a .deb package into the given directory."""
76+
_run_command(["dpkg-deb", "-x", str(package_path), str(extract_dir)])
77+
78+
79+
def _extract_rpm(package_path: Path, extract_dir: Path) -> None:
80+
"""Extract a .rpm package into the given directory."""
81+
with subprocess.Popen(
82+
["rpm2cpio", str(package_path)], stdout=subprocess.PIPE
83+
) as rpm2cpio:
84+
_run_command(["cpio", "-idm"], stdin=rpm2cpio.stdout, cwd=extract_dir)
85+
rpm2cpio.wait()
86+
if rpm2cpio.returncode != 0:
87+
raise subprocess.CalledProcessError(
88+
rpm2cpio.returncode, ["rpm2cpio", str(package_path)]
89+
)
90+
91+
92+
def python_dev_headers(
93+
version: str, storage_dir: Path, uri_override: str | None = None
94+
) -> Path:
95+
"""
96+
Return the path to the include directory containing Python headers.
97+
98+
Downloads the appropriate python-dev/python-devel package for the current Linux
99+
distribution if necessary, extracts it, and returns the path to the
100+
extracted headers.
101+
102+
Args:
103+
version: The Python version string, e.g. "3.11".
104+
storage_dir: The directory to use for downloading and extracting packages.
105+
uri_override: If provided, this URI will be used instead of determining
106+
the package URL based on the distribution. This is intended for testing.
107+
108+
Returns:
109+
The path to the directory containing Python.h and other headers.
110+
111+
Raises:
112+
ValueError: If the current distribution is not supported.
113+
subprocess.CalledProcessError: If downloading or extracting the package fails.
114+
FileNotFoundError: If Python.h cannot be found after extraction.
115+
"""
116+
download_dir = storage_dir / "packages"
117+
download_dir.mkdir(exist_ok=True)
118+
119+
uri = uri_override or _get_download_package_urls(version)
120+
121+
# Fetch the package file to the download directory.
122+
package_path = download_dir / uri.split("/")[-1]
123+
if not package_path.exists():
124+
_run_command(["curl", "-L", "-o", str(package_path), uri])
125+
126+
extract_dir = storage_dir / (package_path.stem + "_extracted")
127+
if not extract_dir.exists():
128+
if package_path.suffix == ".deb":
129+
extract = _extract_deb
130+
elif package_path.suffix == ".rpm":
131+
extract = _extract_rpm
132+
else:
133+
raise ValueError(
134+
f"""Unknown package format: {package_path.suffix}. Expected .deb or .rpm."""
135+
)
136+
137+
extract_dir.mkdir(exist_ok=True)
138+
extract(package_path, extract_dir)
139+
140+
return extract_dir

‎python/src/ubeacon/extension/ubeacon.py‎

Lines changed: 84 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import functools
1313
import json
1414
import os
15+
import re
1516
import subprocess
1617
import tempfile
1718
from pathlib import Path
@@ -23,9 +24,11 @@
2324
import pygments.formatters
2425
import pygments.lexers
2526
from src.udbpy import locations, report # pyright: ignore[reportMissingModuleSource]
26-
from src.udbpy.gdb_extensions import gdbutils # pyright: ignore[reportMissingModuleSource]
27+
from src.udbpy.gdb_extensions import (
28+
gdbutils,
29+
) # pyright: ignore[reportMissingModuleSource]
2730

28-
from . import debuggee, messages
31+
from . import debuggee, download_python_headers, messages
2932

3033
PREFIX = "s_ubeacon"
3134
STATE_STRUCT = PREFIX
@@ -36,6 +39,83 @@
3639
EXCEPTION_FN = f"{TRACE_PREFIX}_exception"
3740

3841

42+
def wrap_build(python_executable: str, cache_dir: Path, addon_root: Path) -> None:
43+
"""Build the UBeacon library using the specified Python executable.
44+
45+
If the Python development headers are not available on the system, they are
46+
downloaded using the distro's package manager.
47+
48+
python_executable:
49+
The path to the Python executable to build against.
50+
cache_dir:
51+
The directory to use for caching build artifacts and downloaded headers.
52+
addon_root:
53+
The root directory of the addon, where `setup.py` is located.
54+
"""
55+
56+
version_output = subprocess.check_output(
57+
[python_executable, "--version"], text=True, cwd=addon_root
58+
).strip()
59+
m = re.match(r"Python (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)", version_output)
60+
if not m:
61+
raise report.ReportableError(
62+
f"Unexpected output from '{python_executable} --version': {version_output}"
63+
)
64+
version_string = f"{m['major']}.{m['minor']}"
65+
66+
# Does this Python executable have development headers installed already?
67+
include_paths = subprocess.check_output(
68+
[
69+
python_executable,
70+
"-c",
71+
"import sysconfig; print(sysconfig.get_path('include'))",
72+
],
73+
text=True,
74+
).strip()
75+
76+
if not Path(include_paths).exists():
77+
# Fetch headers from a distro package
78+
header_dir = download_python_headers.python_dev_headers(
79+
version_string, storage_dir=cache_dir
80+
)
81+
include_paths = f"{header_dir}/usr/include:{header_dir}/usr/include/python{version_string}"
82+
83+
try:
84+
subprocess.run(
85+
[
86+
python_executable,
87+
"setup.py",
88+
"build",
89+
"--quiet",
90+
f"--build-base={cache_dir}",
91+
],
92+
text=True,
93+
cwd=addon_root,
94+
check=True,
95+
capture_output=True,
96+
env={**os.environ, "PYTHON_INCLUDE_DIRS": include_paths},)
97+
98+
except subprocess.CalledProcessError as exc:
99+
output = ""
100+
if exc.output:
101+
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
102+
tf.write(exc.output)
103+
tf.flush()
104+
output += f"Saved stdout to {tf.name}.\n"
105+
if exc.stderr:
106+
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
107+
tf.write(exc.stderr)
108+
tf.flush()
109+
output += f"Saved stderr to {tf.name}.\n"
110+
raise report.ReportableError(
111+
f"""Error occurred in Python: could not debug this version of Python.
112+
113+
You may need to install Python development headers for this version.
114+
115+
{output}"""
116+
)
117+
118+
39119
@functools.cache
40120
def build() -> Path:
41121
"""
@@ -81,40 +161,8 @@ def build() -> Path:
81161
return lib_path
82162

83163
# Not found, build it
84-
try:
85-
subprocess.run(
86-
[
87-
python_executable,
88-
"setup.py",
89-
"build",
90-
"--quiet",
91-
f"--build-base={cache_dir}",
92-
],
93-
text=True,
94-
cwd=root,
95-
check=True,
96-
capture_output=True,
97-
)
98-
except subprocess.CalledProcessError as exc:
99-
if exc.output:
100-
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
101-
tf.write(exc.output)
102-
tf.flush()
103-
report.user(
104-
f"Saved stdout to {tf.name}.\n"
105-
)
106-
if exc.stderr:
107-
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
108-
tf.write(exc.stderr)
109-
tf.flush()
110-
report.user(
111-
f"Saved stderr to {tf.name}.\n"
112-
)
113-
raise report.ReportableError(
114-
"""Error occurred in Python: could not debug this version of Python.
115-
116-
You may need to install Python development headers for this version."""
117-
)
164+
wrap_build(python_executable, cache_dir, root)
165+
118166
output = subprocess.check_output(
119167
[python_executable, "find_so.py", cache_dir], text=True, cwd=root
120168
)

0 commit comments

Comments
 (0)