From 73ffa82eec7bcfa3270fb055fb769c02a1c98d87 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Thu, 26 Feb 2026 10:53:07 -0600 Subject: [PATCH 1/2] JCF: Issue #341: install Python packages even if there are no C++ packages; also, improve Python installation benchmark messaging --- bin/dbt-build | 117 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 47 deletions(-) diff --git a/bin/dbt-build b/bin/dbt-build index ce88253..e6ea744 100755 --- a/bin/dbt-build +++ b/bin/dbt-build @@ -166,6 +166,58 @@ if not os.path.exists(BUILDDIR): except PermissionError: error(f"You don't have permission to create {BUILDDIR} from this directory. Exiting...") +stringio_obj2 = io.StringIO() +sh.date(_out=stringio_obj2) +datestring=re.sub("[: ]+", "_", stringio_obj2.getvalue().strip()) + +build_log=f"{LOGDIR}/build_attempt_{datestring}.log" + +pyinstalltime=None +python_packages=get_package_list(PYTHONDIR) + +if python_packages: + print(f"\nPython packages found in {PYTHONDIR}, will proceed with their installation...") + starttime_pyinstall_d=get_time("as_date") + starttime_pyinstall_s=get_time("as_seconds_since_epoch") + + pyinstall_success=True + for pypkg in get_package_list(PYTHONDIR): + if not os.path.exists(f"{PYTHONDIR}/{pypkg}/pyproject.toml"): + error(f"""Unable to find {PYTHONDIR}/{pypkg}/pyproject.toml; + note that modern standards for DUNE-DAQ Python packages are expected. Exiting... + """) + + fullcmd=f"pip install {PYTHONDIR}/{pypkg}" + rich.print(f"Executing '{fullcmd}'") + retval=pytee.run(fullcmd.split(" ")[0], fullcmd.split(" ")[1:], build_log) + if retval != 0: + pyinstall_success=False + break + + endtime_pyinstall_d=get_time("as_date") + endtime_pyinstall_s=get_time("as_seconds_since_epoch") + + if pyinstall_success: + pyinstalltime=int(endtime_pyinstall_s) - int(starttime_pyinstall_s) + else: + error(f""" + + This script ran into a problem installing a Python package from {PYTHONDIR}. + Scroll up for details or look at the build log via + + less -R {build_log} + + Exiting... + """) +else: + rich.print(f""" + +No Python repos have been found in {PYTHONDIR}, +so no Python package installation will take place. + +""") + + os.chdir(BUILDDIR) if args.clean_build or args.codegen_only: @@ -207,21 +259,22 @@ def erase_installdir_contents(): else: error(f"Installation directory is defined as \"{INSTALLDIR}\", which would result in the deletion of the entire contents of this system if it weren't for this check!!!") - if not get_package_list(SRCDIR): - print(f"""No package repos have been found in {SRCDIR}, + + if pyinstalltime is not None: + rich.print("") + rich.print(f"Python package installation stage took {pyinstalltime} seconds") + rich.print(f"Start time: {starttime_pyinstall_d}") + rich.print(f"End time: {endtime_pyinstall_d}") + + print("") + print(f"""No C++ package repos have been found in {SRCDIR}, so no C++ build will take place. However, a script saving the environment will -be put in {INSTALLDIR}""") +be put in {INSTALLDIR}\n""") erase_installdir_contents() create_app_rte_script() sys.exit(0) -stringio_obj2 = io.StringIO() -sh.date(_out=stringio_obj2) -datestring=re.sub("[: ]+", "_", stringio_obj2.getvalue().strip()) - -build_log=f"{LOGDIR}/build_attempt_{datestring}.log" - cmake="cmake" if args.cmake_trace: cmake = f"{cmake} --trace" @@ -415,39 +468,6 @@ for pkg in get_package_list(BUILDDIR): with open(f"{INSTALLDIR}/build_summary_info.json", 'w') as sbi_f: json.dump( summary_build_info, sbi_f, sort_keys=True, indent=4 ) -starttime_pyinstall_d=get_time("as_date") -starttime_pyinstall_s=get_time("as_seconds_since_epoch") - -pyinstall_success=True -for pypkg in get_package_list(PYTHONDIR): - if not os.path.exists(f"{PYTHONDIR}/{pypkg}/pyproject.toml"): - error(f"""Unable to find {PYTHONDIR}/{pypkg}/pyproject.toml; -note that modern standards for DUNE-DAQ Python packages are expected. Exiting... -""") - - fullcmd=f"pip install {PYTHONDIR}/{pypkg}" - rich.print(f"Executing '{fullcmd}'") - retval=pytee.run(fullcmd.split(" ")[0], fullcmd.split(" ")[1:], build_log) - if retval != 0: - pyinstall_success=False - break - -endtime_pyinstall_d=get_time("as_date") -endtime_pyinstall_s=get_time("as_seconds_since_epoch") - -if pyinstall_success: - pyinstalltime=int(endtime_pyinstall_s) - int(starttime_pyinstall_s) -else: - error(f""" - -This script ran into a problem installing a Python package from {PYTHONDIR}. -Scroll up for details or look at the build log via - -less -R {build_log} - -Exiting... -""") - unittest_overviews = [] if run_tests: stringio_obj5 = io.StringIO() @@ -638,6 +658,14 @@ if args.lint: if retval != 0: error(f"There was a problem linting the file \"{code_to_lint}\". Exiting...") +rich.print("") +if pyinstalltime is not None: + rich.print(f"Python package installation stage took {pyinstalltime} seconds") + rich.print(f"Start time: {starttime_pyinstall_d}") + rich.print(f"End time: {endtime_pyinstall_d}") +else: + rich.print(f"Python package installation stage skipped as no Python repos were found in {PYTHONDIR}") + rich.print("") if cfggentime is not None: rich.print(f"CMake's build file config+generate stages took {cfggentime} seconds") @@ -651,11 +679,6 @@ rich.print(f"CMake's build+install stages took {buildtime} seconds") rich.print(f"Start time: {starttime_build_d}") rich.print(f"End time: {endtime_build_d}") -rich.print("") -rich.print(f"Python package installation stage took {pyinstalltime} seconds") -rich.print(f"Start time: {starttime_pyinstall_d}") -rich.print(f"End time: {endtime_pyinstall_d}") - if num_estimated_warnings == 0: pass # Avoiding screen clutter more important than making developers feel good From 770bc82220da7cd33a8583837c91c346e0658549 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Thu, 26 Feb 2026 12:41:15 -0600 Subject: [PATCH 2/2] JCF: Issue #341: add a test to test_daq-buildtools.sh which tests for the thing this Issue is fixing --- scripts/test_daq-buildtools.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/test_daq-buildtools.sh b/scripts/test_daq-buildtools.sh index 4a982d9..a75b1d1 100755 --- a/scripts/test_daq-buildtools.sh +++ b/scripts/test_daq-buildtools.sh @@ -30,6 +30,7 @@ trap cleanup EXIT SIGINT SIGTERM release="last_fddaq" repo="ipm" +pyrepo="daqpytools" dbt_branch="develop" while [[ $# -gt 0 ]]; do @@ -67,7 +68,8 @@ fi echo -e "Running daq-buildtools commands using:\n" echo -e "\tRelease name: $release" echo -e "\tdbt branch: $dbt_branch" -echo -e "\trepo: $repo\n" +echo -e "\trepo: $repo" +echo -e "\tpyrepo: $pyrepo\n" . /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh || exit 1 setup_dbt latest_v5 || exit 2 @@ -93,12 +95,24 @@ rm -f dbt-setup-release_result.txt echo "*********************************TEST dbt-create ***************************************" dbt-create -s ${extra_args[@]} $release || exit 5 cd $(ls) # Only thing in the directory will be the work area + + +cd pythoncode +git clone https://github.com/DUNE-DAQ/$pyrepo || exit 16 +cd .. +. env.sh || exit 17 +rm -f .venv/lib64/python*/site-packages/$pyrepo/__init__.py || exit 18 +echo "******************************TEST dbt-build (Python) *************************************" +dbt-build || exit 19 +find .venv/lib64/python*/site-packages/$pyrepo/__init__.py | read || exit 20 +rm -rf pythoncode/$pyrepo + cd sourcecode git clone https://github.com/DUNE-DAQ/$repo || exit 6 cd .. . env.sh || exit 7 -echo "**********************************TEST dbt-build ****************************************" +echo "******************************TEST dbt-build (C++) *************************************" dbt-build || exit 8 echo "******************************TEST dbt-build --unittest *********************************"