diff --git a/.black b/.black deleted file mode 100644 index 718d6bded9..0000000000 --- a/.black +++ /dev/null @@ -1,12 +0,0 @@ -[tool.black] -line-length = 120 -target-version = ['py37', 'py38', 'py39', 'py310'] -force-exclude = ''' -( - /( - | \.git - | \.github - )/ - | libensemble/files/to/avoid -) -''' diff --git a/.codecov.yml b/.codecov.yml index 18ef408010..bf866b089c 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -3,5 +3,4 @@ ignore: - "libensemble/tools/forkable_pdb.py" - "libensemble/tools/live_data/*" - "libensemble/sim_funcs/executor_hworld.py" - - "libensemble/gen_funcs/persistent_ax_multitask.py" - - "libensemble/gen_funcs/persistent_gpCAM.py" + - "libensemble/gen_funcs/persistent_tasmanian.py" diff --git a/.flake8 b/.flake8 index d49bc0d3bc..5dabea081d 100644 --- a/.flake8 +++ b/.flake8 @@ -46,6 +46,3 @@ per-file-ignores = libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py:E402 libensemble/tests/unit_tests/test_persistent_aposmm.py:E402 libensemble/tests/unit_tests/RENAME_test_persistent_aposmm.py:E402 - - # Allow undefined name '__version__' - setup.py:F821 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 66ac4c3f1c..21ca389e09 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -36,4 +36,3 @@ updates: submodule-updates: patterns: - "*" - diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index b694113d88..7a008367da 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -17,29 +17,29 @@ jobs: matrix: os: [ubuntu-latest] mpi-version: [mpich] - python-version: [3.9, "3.10", "3.11", "3.12", "3.13"] - pydantic-version: ["2.8.2"] + python-version: ["3.10", "3.11", "3.12", "3.13"] + pydantic-version: ["2.10.6"] comms-type: [m, l] include: - os: macos-latest python-version: "3.11" mpi-version: mpich - pydantic-version: "2.8.2" + pydantic-version: "2.10.6" comms-type: m - os: macos-latest python-version: "3.11" mpi-version: mpich - pydantic-version: "2.8.2" + pydantic-version: "2.10.6" comms-type: l - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.17" + pydantic-version: "1.10.21" comms-type: m - os: ubuntu-latest mpi-version: mpich python-version: "3.10" - pydantic-version: "1.10.17" + pydantic-version: "1.10.21" comms-type: l env: @@ -129,4 +129,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.28.2 + - uses: crate-ci/typos@v1.31.1 diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 773dac1541..e9b0275dda 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -11,43 +11,43 @@ jobs: matrix: os: [ubuntu-latest] mpi-version: [mpich] - python-version: [3.9, "3.10", "3.11", "3.12"] - pydantic-version: ["2.8.2"] + python-version: ['3.10', '3.11', '3.12', '3.13'] + pydantic-version: ['2.10.6'] comms-type: [m, l] include: - os: macos-latest - python-version: 3.11 + python-version: '3.13' mpi-version: mpich - pydantic-version: "2.8.2" + pydantic-version: '2.10.6' comms-type: m - os: macos-latest - python-version: 3.11 + python-version: '3.13' mpi-version: mpich - pydantic-version: "2.8.2" + pydantic-version: '2.10.6' comms-type: l - os: ubuntu-latest - python-version: "3.10" + python-version: '3.12' mpi-version: mpich - pydantic-version: "2.8.2" + pydantic-version: '2.10.6' comms-type: t - os: ubuntu-latest - mpi-version: "openmpi" - pydantic-version: "2.8.2" - python-version: "3.12" + mpi-version: 'openmpi' + pydantic-version: '2.10.6' + python-version: '3.12' comms-type: l - os: ubuntu-latest mpi-version: mpich - python-version: "3.10" - pydantic-version: "1.10.17" + python-version: '3.12' + pydantic-version: '1.10.21' comms-type: m - os: ubuntu-latest mpi-version: mpich - python-version: "3.10" - pydantic-version: "1.10.17" + python-version: '3.12' + pydantic-version: '1.10.21' comms-type: l env: - HYDRA_LAUNCHER: "fork" + HYDRA_LAUNCHER: 'fork' TERM: xterm-256color GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -61,7 +61,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: activate-environment: condaenv - miniconda-version: "latest" + miniconda-version: 'latest' python-version: ${{ matrix.python-version }} channels: conda-forge channel-priority: flexible @@ -75,8 +75,8 @@ jobs: - name: Install Ubuntu compilers if: matrix.os == 'ubuntu-latest' run: | - conda install gcc_linux-64 - pip install nlopt==2.9.0 + conda install gcc_linux-64 + pip install nlopt==2.9.0 # Roundabout solution on macos for proper linking with mpicc - name: Install macOS compilers @@ -93,25 +93,25 @@ jobs: run: | conda env update --file install/gen_deps_environment.yml - - name: Install gpcam - if: matrix.python-version != '3.12' + - name: Install gpcam and octave # Neither yet support 3.13 + if: matrix.python-version <= '3.12' run: | pip install gpcam + conda install octave - - name: Install surmise + - name: Install surmise and Tasmanian if: matrix.os == 'ubuntu-latest' run: | pip install --upgrade git+https://github.com/bandframework/surmise.git + pip install Tasmanian --user - name: Install generator dependencies for Ubuntu tests - if: matrix.os == 'ubuntu-latest' && matrix.python-version != '3.12' + if: matrix.os == 'ubuntu-latest' && matrix.python-version <= '3.12' run: | - sudo apt-get install bc - pip install -r install/ubuntu_no312.txt - pip install Tasmanian --user + pip install scikit-build packaging - name: Install Balsam on Pydantic 1 - if: matrix.pydantic-version == '1.10.17' + if: matrix.pydantic-version == '1.10.21' run: | conda install pyzmq git clone https://github.com/argonne-lcf/balsam.git @@ -120,10 +120,10 @@ jobs: - name: Install other testing dependencies run: | - conda install octave pip install -r install/testing_requirements.txt pip install -r install/misc_feature_requirements.txt source install/install_ibcdfo.sh + conda install numpy scipy - name: Install libEnsemble, flake8, lock environment run: | @@ -131,27 +131,26 @@ jobs: pip install -e . flake8 libensemble - - name: Remove test for persistent Tasmanian, Surmise on Python 3.12 - if: matrix.python-version >= '3.12' + - name: Remove test using octave, gpcam on Python 3.13 + if: matrix.python-version >= '3.13' run: | - rm ./libensemble/tests/regression_tests/test_persistent_tasmanian.py - rm ./libensemble/tests/regression_tests/test_persistent_tasmanian_async.py - rm ./libensemble/tests/regression_tests/test_persistent_surmise_calib.py - rm ./libensemble/tests/regression_tests/test_persistent_surmise_killsims.py + rm ./libensemble/tests/regression_tests/test_persistent_fd_param_finder.py # needs octave, which doesn't yet support 3.13 + rm ./libensemble/tests/regression_tests/test_persistent_aposmm_external_localopt.py # needs octave, which doesn't yet support 3.13 + rm ./libensemble/tests/regression_tests/test_gpCAM.py # needs gpcam, which doesn't build on 3.13 - name: Install redis/proxystore on Pydantic 2 - if: matrix.pydantic-version == '2.8.2' + if: matrix.pydantic-version == '2.10.6' run: | pip install redis pip install proxystore==0.7.0 - name: Remove proxystore test on Pydantic 1 - if: matrix.pydantic-version == '1.10.17' + if: matrix.pydantic-version == '1.10.21' run: | rm ./libensemble/tests/regression_tests/test_proxystore_integration.py - name: Remove Balsam/Globus-compute tests on Pydantic 2 - if: matrix.pydantic-version == '2.8.2' + if: matrix.pydantic-version == '2.10.6' run: | rm ./libensemble/tests/unit_tests/test_ufunc_runners.py rm ./libensemble/tests/unit_tests/test_executor_balsam.py @@ -181,4 +180,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.28.2 + - uses: crate-ci/typos@v1.31.1 diff --git a/.gitignore b/.gitignore index 828a6fff61..c6bd3c0ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ dist/ .spyproject/ .hypothesis +.pixi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dce6886e14..ab7687d1ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer exclude: ^(.*\.xml|.*\.svg)$ @@ -8,25 +8,25 @@ repos: exclude: ^(.*\.xml|.*\.svg)$ - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 6.0.0 hooks: - id: isort args: [--profile=black, --line-length=120] - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 25.1.0 hooks: - id: black args: [--line-length=120] - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 7.1.1 hooks: - id: flake8 args: [--max-line-length=120] - repo: https://github.com/asottile/blacken-docs - rev: v1.12.1 + rev: 1.19.1 hooks: - id: blacken-docs additional_dependencies: [black==22.12.0] diff --git a/.readthedocs.yml b/.readthedocs.yml index a8caf706f0..50b0395fae 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ version: 2 build: os: "ubuntu-22.04" tools: - python: "3.9" + python: "3.10" sphinx: configuration: docs/conf.py diff --git a/.typos.toml b/.typos.toml deleted file mode 100644 index 6f83997492..0000000000 --- a/.typos.toml +++ /dev/null @@ -1,21 +0,0 @@ -[default] -extend-ignore-identifiers-re = [ - ".*NDArray.*", - "8ba9de56.*" -] - -[default.extend-words] -als = "als" -datas = "datas" -numer = "numer" -inout = "inout" -arange = "arange" -wrk = "wrk" -EOF = "EOF" -HPE = "HPE" -RO = "RO" -lst = "lst" -noy = "noy" - -[files] -extend-exclude = ["*.bib", "*.xml", "docs/nitpicky"] diff --git a/.wci.yml b/.wci.yml index 78b37075ff..f03aa0c635 100644 --- a/.wci.yml +++ b/.wci.yml @@ -16,8 +16,8 @@ description: | language: Python release: - version: 1.4.3 - date: 2024-12-16 + version: 1.5.0 + date: 2025-04-10 documentation: general: https://libensemble.readthedocs.io diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b81a395af8..bbb2bee549 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,46 @@ GitHub issues are referenced, and can be viewed with hyperlinks on the `github r .. _`github releases page`: https://github.com/Libensemble/libensemble/releases +Release 1.5.0 +-------------- + +:Date: Apr 10, 2025 + +General Updates: + +* Migrate package build system to `pyproject.toml` (with `pixi` support). #1459 +* Improve handling when no MPI found. #1514 +* `ensemble.save_output()` can save without appending attributes `append_attrs=False`. #1531 +* Improve handling of worker-specific `persis_info` fields when they are not initially provided. #1531 + * Bugfix: Fix `final_gen_send` when there are no worker-specific `persis_info` fields. + * Handle worker-generated `persis_info` fields. + * Ensure `persis_info` is initialized to an empty dictionary in user functions instead of `None`. + +Examples: + +* Update Ax generator for `Ax v0.5.0`. #1508 +* Rename gpCAM generators. #1516 + * `persistent_gpCAM_ask_tell` to `persistent_gpCAM` + * `persistent_gpCAM_simple` to `persistent_gpCAM_covar` (in fact less simple) +* Persistent generators return `None` as first return value unless `H_o` is updated. #1515 +* Add LUMI to known platforms. #1546 + +Documentation: + +* Revamp Examples and HPC section of documentation. #1501, #1536, #1539 +* Add tutorial and notebook demonstrating surrogate model creation with gpCAM. #1531 +* Update Aurora guide. #1510 +* Update and documented APOSMM/WarpX example. #1543 + +:Note: + +* Tests were run on Linux and MacOS with Python versions 3.10, 3.11, 3.12, 3.13 +* Heterogeneous workflows tested on Aurora (ALCF), Polaris (ALCF), LUMI (EuroHPC JU), and Perlmutter (NERSC). + +:Known Issues: + +* See known issues section in the documentation. + Release 1.4.3 -------------- diff --git a/LICENSE b/LICENSE index de0d1ca0ea..6a45c6a4cf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2018-2024, UChicago Argonne, LLC and the libEnsemble Development Team +Copyright (c) 2018-2025, UChicago Argonne, LLC and the libEnsemble Development Team All Rights Reserved. Redistribution and use in source and binary forms, with or without diff --git a/MANIFEST.in b/MANIFEST.in index 4127196957..c67c0b894c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +include pyproject.toml include setup.py include *.cfg include *.rst diff --git a/README.rst b/README.rst index 1a27ec7484..df5fa587e4 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,8 @@ libEnsemble: A complete toolkit for dynamic ensembles of calculations ===================================================================== -Adaptive, portable, and scalable software for connecting "deciders" to experiments or simulations. +libEnsemble empowers model-driven ensembles to solve design, decision, +and inference problems on the world's leading supercomputers such as Frontier, Aurora, and Perlmutter. • **Dynamic ensembles**: Generate parallel tasks on-the-fly based on previous computations. • **Extreme portability and scaling**: Run on or across laptops, clusters, and leadership-class machines. @@ -19,8 +20,6 @@ Adaptive, portable, and scalable software for connecting "deciders" to experimen • **Data-flow between tasks**: Running ensemble members can send and receive data. • **Low start-up cost**: No additional background services or processes required. -libEnsemble is effective at solving design, decision, and inference problems on parallel resources. - `Quickstart`_ Installation @@ -96,8 +95,10 @@ Try some other examples live in Colab. +---------------------------------------------------------------+-------------------------------------+ | Optimization example that finds multiple minima. | |Optimization example| | +---------------------------------------------------------------+-------------------------------------+ +| Surrogate model generation with gpCAM. | |Surrogate Modeling| | ++---------------------------------------------------------------+-------------------------------------+ -There are many more examples in the `regression tests`_. +There are many more examples in the `regression tests`_ and `Community Examples repository`_. Resources ========= @@ -178,3 +179,6 @@ Resources .. |Optimization example| image:: https://colab.research.google.com/assets/colab-badge.svg :target: http://colab.research.google.com/github/Libensemble/libensemble/blob/develop/examples/tutorials/aposmm/aposmm_tutorial_notebook.ipynb + +.. |Surrogate Modeling| image:: https://colab.research.google.com/assets/colab-badge.svg + :target: https://colab.research.google.com/github/Libensemble/libensemble/blob/develop/examples/tutorials/gpcam_surrogate_model/gpcam.ipynb diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 2397da52f9..b8a3ea2ce5 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -90,7 +90,7 @@ Common Errors .. dropdown:: **PETSc and MPI errors with "[unset]: write_line error; fd=-1 buf=:cmd=abort exitcode=59"** - with ``python [test with PETSc].py --comms local --nworkers 4`` + with ``python [test with PETSc].py --nworkers 4`` This error occurs on some platforms when using PETSc with libEnsemble in ``local`` (multiprocessing) mode. We believe this is due to PETSc initializing MPI diff --git a/docs/advanced_installation.rst b/docs/advanced_installation.rst index 758b7277a0..ad3131ee15 100644 --- a/docs/advanced_installation.rst +++ b/docs/advanced_installation.rst @@ -6,10 +6,10 @@ libEnsemble can be installed from ``pip``, ``Conda``, or ``Spack``. libEnsemble requires the following dependencies, which are typically automatically installed alongside libEnsemble: -* Python_ ``>= 3.9`` +* Python_ ``>= 3.10`` * NumPy_ ``>= 1.21`` * psutil_ ``>= 5.9.4`` -* `pydantic`_ ``<= 1.10.12`` +* `pydantic`_ ``>= 1.10.12`` * pyyaml_ ``>= v6.0`` * tomli_ ``>= 1.2.1`` @@ -139,7 +139,7 @@ Further recommendations for selected HPC systems are given in the **Hint**: When combining Spack and Conda, you can access your Conda Python and packages in your ``~/.spack/packages.yaml`` while your Conda environment is activated, using ``CONDA_PREFIX`` - For example, if you have an activated Conda environment with Python 3.9 and SciPy installed: + For example, if you have an activated Conda environment with Python 3.10 and SciPy installed: .. code-block:: yaml @@ -152,12 +152,12 @@ Further recommendations for selected HPC systems are given in the py-numpy: externals: - spec: "py-numpy" - prefix: $CONDA_PREFIX/lib/python3.9/site-packages/numpy + prefix: $CONDA_PREFIX/lib/python3.10/site-packages/numpy buildable: False py-scipy: externals: - spec: "py-scipy" - prefix: $CONDA_PREFIX/lib/python3.9/site-packages/scipy + prefix: $CONDA_PREFIX/lib/python3.10/site-packages/scipy buildable: True For more information on Spack builds and any particular considerations diff --git a/docs/conf.py b/docs/conf.py index bad6e4246d..7686b741f8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,35 +31,34 @@ def __getattr__(cls, name): return MagicMock() +autodoc_mock_imports = ["ax", "balsam", "gpcam", "IPython", "matplotlib", "pandas", "scipy", "surmise"] + MOCK_MODULES = [ "argparse", "dfols", - "IPython", - "IPython.display", - "IPython.core", - "IPython.core.pylabtools", "math", - "matplotlib", - "matplotlib.pyplot", "mpi4py", "mpmath", "nlopt", "PETSc", "petsc4py", "psutil", - "scipy", - "scipy.io", - "scipy.sparse", - "scipy.spatial", - "scipy.spatial.distance", - "scipy.stats", - "surmise.calibration", - "surmise.emulation", + "pyre_extensions", "Tasmanian", + "torch", ] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) + +class AxParameterWarning(Warning): # Ensure it's a real warning subclass + pass + + +# Fix only `AxParameterWarning` while keeping `ax` mocked +sys.modules["ax.exceptions.core"] = MagicMock() +sys.modules["ax.exceptions.core"].AxParameterWarning = AxParameterWarning + # from libensemble import * # from libensemble.alloc_funcs import * # from libensemble.gen_funcs import * @@ -78,6 +77,7 @@ def __getattr__(cls, name): sys.path.append(os.path.abspath("../libensemble/tools/live_data")) sys.path.append(os.path.abspath("../libensemble/executors")) sys.path.append(os.path.abspath("../libensemble/resources")) +sys.path.append(os.path.abspath("../libensemble/tests/scaling_tests/forces")) # print(sys.path) # -- General configuration ------------------------------------------------ @@ -131,7 +131,6 @@ def __getattr__(cls, name): autodoc_pydantic_model_show_json = False -autodoc_mock_imports = ["balsam"] extlinks = { "duref": ("http://docutils.sourceforge.net/docs/ref/rst/" "restructuredtext.html#%s", ""), "durole": ("http://docutils.sourceforge.net/docs/ref/rst/" "roles.html#%s", ""), @@ -230,8 +229,14 @@ def __getattr__(cls, name): # html_static_path = [] +def remove_noqa(app, what, name, obj, options, lines): + for i, line in enumerate(lines): + lines[i] = line.replace("# noqa", "") + + def setup(app): app.add_css_file("my_theme.css") + app.connect("autodoc-process-docstring", remove_noqa) # Custom sidebar templates, must be a dictionary that maps document names diff --git a/docs/data_structures/data_structures.rst b/docs/data_structures/data_structures.rst index 1d88d993dd..35a5ba0158 100644 --- a/docs/data_structures/data_structures.rst +++ b/docs/data_structures/data_structures.rst @@ -8,9 +8,9 @@ See :ref:`here` for instruction on constructing a complete workflow :maxdepth: 2 :caption: libEnsemble Specifications: - sim_specs - gen_specs libE_specs + gen_specs + sim_specs alloc_specs platform_specs persis_info diff --git a/docs/data_structures/libE_specs.rst b/docs/data_structures/libE_specs.rst index 2a9195cea5..c0ca141403 100644 --- a/docs/data_structures/libE_specs.rst +++ b/docs/data_structures/libE_specs.rst @@ -1,7 +1,7 @@ .. _datastruct-libe-specs: -General Specs -============= +LibE Specs +========== libEnsemble is primarily customized by setting options within a ``LibeSpecs`` class or dictionary. @@ -10,11 +10,10 @@ libEnsemble is primarily customized by setting options within a ``LibeSpecs`` cl from libensemble.specs import LibeSpecs specs = LibeSpecs( - comm=MPI.COMM_WORLD, - comms="mpi", - save_every_k_gens=1000, + gen_on_manager=True, + save_every_k_gens=100, sim_dirs_make=True, - ensemble_dir_path="/scratch/ensemble", + nworkers=4 ) .. dropdown:: Settings by Category @@ -272,8 +271,8 @@ libEnsemble is primarily customized by setting options within a ``LibeSpecs`` cl ``disable_resource_manager`` is set. **dedicated_mode** [bool] = ``False``: - Disallow any resources running libEnsemble processes (manager and workers) - from being valid targets for app submissions. + Instructs libEnsemble’s MPI executor not to run applications on nodes where + libEnsemble processes (manager and workers) are running. **zero_resource_workers** [list of ints]: List of workers (by IDs) that require no resources. For when a fixed mapping of workers diff --git a/docs/dev_guide/release_management/release_process.rst b/docs/dev_guide/release_management/release_process.rst index fa3427f9c7..60d5b80cd6 100644 --- a/docs/dev_guide/release_management/release_process.rst +++ b/docs/dev_guide/release_management/release_process.rst @@ -21,7 +21,7 @@ Before release - Year in ``LICENSE`` is checked for correctness. (Note: The year generated in docs by ``docs/conf.py`` should be automatic). -- ``setup.py`` and ``libensemble/__init__.py`` are checked to ensure all +- ``pyproject.toml`` and ``libensemble/__init__.py`` are checked to ensure all information is up to date. - Update ``.wci.yml`` in root directory (version, date and any other diff --git a/docs/examples/alloc_funcs.rst b/docs/examples/alloc_funcs.rst index f9259d27aa..f54f4bf3c4 100644 --- a/docs/examples/alloc_funcs.rst +++ b/docs/examples/alloc_funcs.rst @@ -5,11 +5,15 @@ Allocation Functions Below are example allocation functions available in libEnsemble. +Many users use these unmodified. + .. IMPORTANT:: See the API for allocation functions :ref:`here`. .. note:: - The default allocation function is ``give_sim_work_first``. + The default allocation function (for non-persistent generators) is :ref:`give_sim_work_first`. + + The most commonly used (for persistent generators) is :ref:`start_only_persistent`. .. role:: underline :class: underline @@ -59,3 +63,39 @@ start_persistent_local_opt_gens .. automodule:: start_persistent_local_opt_gens :members: :undoc-members: + +fast_alloc_and_pausing +---------------------- +.. automodule:: fast_alloc_and_pausing + :members: + :undoc-members: + +only_one_gen_alloc +------------------ +.. automodule:: only_one_gen_alloc + :members: + :undoc-members: + +start_fd_persistent +------------------- +.. automodule:: start_fd_persistent + :members: + :undoc-members: + +persistent_aposmm_alloc +----------------------- +.. automodule:: persistent_aposmm_alloc + :members: + :undoc-members: + +give_pregenerated_work +---------------------- +.. automodule:: give_pregenerated_work + :members: + :undoc-members: + +inverse_bayes_allocf +-------------------- +.. automodule:: inverse_bayes_allocf + :members: + :undoc-members: diff --git a/docs/examples/aposmm.rst b/docs/examples/aposmm.rst index 9ecdbdb578..f5a7a2f11c 100644 --- a/docs/examples/aposmm.rst +++ b/docs/examples/aposmm.rst @@ -2,8 +2,16 @@ APOSMM ------ Asynchronously Parallel Optimization Solver for finding Multiple Minima -(APOSMM) coordinates concurrent local optimization runs in order to identify -many local minima. +(APOSMM) coordinates concurrent local optimization runs to identify +many local minima faster on parallel hardware. + +Supported local optimization routines include: + + - DFO-LS_ Derivative-free solver for (bound constrained) nonlinear least-squares minimization + - NLopt_ Library for nonlinear optimization, providing a common interface for various methods + - `scipy.optimize`_ Open-source solvers for nonlinear problems, linear programming, + constrained and nonlinear least-squares, root finding, and curve fitting. + - `PETSc/TAO`_ Routines for the scalable (parallel) solution of scientific applications Required: mpmath_, SciPy_ @@ -12,32 +20,26 @@ Optional (see below): petsc4py_, nlopt_, DFO-LS_ Configuring APOSMM ^^^^^^^^^^^^^^^^^^ -APOSMM works with a choice of optimizers, some requiring external packages. To -import the optimization packages (and their dependencies) at a global level -(recommended), add the following lines in the calling script before importing -APOSMM:: +APOSMM works with a choice of optimizers, some requiring external packages. Specify +them on a global level before importing APOSMM:: import libensemble.gen_funcs libensemble.gen_funcs.rc.aposmm_optimizers = -where ``optimizers`` is a string (or list of strings) from the available options: +where ``optimizers`` is a string (or list-of-strings) from: ``"petsc"``, ``"nlopt"``, ``"dfols"``, ``"scipy"``, ``"external"`` -.. dropdown:: Issues with ensemble hanging or failed simulations? - - Note that if using **mpi4py** comms, PETSc must be imported at the global - level or the ensemble may hang. +.. dropdown:: Issues with ensemble hanging or failed simulations with PETSc? - Exception: In the case that you are using the MPIExecutor or other MPI inside - a user function and you are using Open MPI, then you must: + If using the MPIExecutor or other MPI routines + and your MPI backend is Open MPI, then you must: - - Use ``local`` comms for libEnsemble (not ``mpi4py``) - - Must **NOT** include the *rc* line above + - Use ``local`` comms for libEnsemble (no ``mpirun``, ``mpiexec``, ``aprun``, etc.). + - Must **NOT** include the *aposmm_optimizers* line above. - This is because PETSc imports MPI, and a global import of PETSc would result - in nested MPI (which is not supported by Open MPI). When the above line is - not used, an import local to the optimization function will happen. + This is because PETSc imports MPI, and a global import of PETSc results + in nested MPI (which is not supported by Open MPI). To see the optimization algorithms supported, see `LocalOptInterfacer`_. @@ -49,13 +51,13 @@ Persistent APOSMM ^^^^^^^^^^^^^^^^^ .. automodule:: persistent_aposmm - :members: + :members: aposmm :undoc-members: LocalOptInterfacer ^^^^^^^^^^^^^^^^^^ .. automodule:: aposmm_localopt_support - :members: + :members: LocalOptInterfacer :undoc-members: .. _DFO-LS: https://github.com/numericalalgorithmsgroup/dfols @@ -63,3 +65,5 @@ LocalOptInterfacer .. _nlopt: https://nlopt.readthedocs.io/en/latest/ .. _petsc4py: https://bitbucket.org/petsc/petsc4py .. _SciPy: https://pypi.org/project/scipy +.. _PETSc/TAO: http://www.mcs.anl.gov/petsc +.. _scipy.optimize: https://docs.scipy.org/doc/scipy/reference/optimize.html diff --git a/docs/examples/ax_multitask.rst b/docs/examples/ax_multitask.rst index e1133866d9..e3984f39a9 100644 --- a/docs/examples/ax_multitask.rst +++ b/docs/examples/ax_multitask.rst @@ -1,16 +1,18 @@ persistent_ax_multitask ----------------------- -Required: Ax_ version <=0.4.0 +Required: `ax-platform`_>=0.5.0 + +Example usage: gp_multitask_ax_ To install:: - pip install "ax-platform<=0.4.0" + pip install ax-platform An example of the Ax multitask GP. .. automodule:: persistent_ax_multitask :members: - :no-undoc-members: -.. _Ax: https://github.com/facebook/Ax +.. _`ax-platform`: https://github.com/facebook/Ax +.. _gp_multitask_ax: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py diff --git a/docs/examples/calling_scripts.rst b/docs/examples/calling_scripts.rst index 64401a5663..183128b6ed 100644 --- a/docs/examples/calling_scripts.rst +++ b/docs/examples/calling_scripts.rst @@ -27,17 +27,22 @@ Electrostatic Forces with Executor These examples are from a test for evaluating the scaling capabilities of libEnsemble by calculating particle electrostatic forces through a user application. This -application is registered with either the MPI or Balsam Executor, then submitted -for execution in the ``sim_f``. Note the use of the ``parse_args()`` and -``save_libE_output()`` convenience functions from the :doc:`tools<../utilities>` module -in the first calling script. +application is registered with the MPIExecutor, then submitted +for execution in the ``sim_f``. Note the use of the ``parse_args=True`` which allows +reading arguments such as the number of workers from the command line. Traditional Version ~~~~~~~~~~~~~~~~~~~ -.. literalinclude:: ../../libensemble/tests/scaling_tests/forces/forces_adv/run_libe_forces.py +Run using five workers with:: + + python run_libe_forces.py -n 5 + +One worker runs a persistent generator and the other four run the forces simulations. + +.. literalinclude:: ../../libensemble/tests/scaling_tests/forces/forces_simple/run_libe_forces.py :language: python - :caption: tests/scaling_tests/forces/forces_adv/run_libe_forces.py + :caption: tests/scaling_tests/forces/forces_simple/run_libe_forces.py :linenos: Object + yaml Version diff --git a/docs/examples/examples_index.rst b/docs/examples/examples_index.rst index 88a2df7a52..1e92e21c03 100644 --- a/docs/examples/examples_index.rst +++ b/docs/examples/examples_index.rst @@ -1,12 +1,10 @@ -Example User Functions and Calling Scripts -========================================== +Overview of Examples +==================== Here we give example generation, simulation, and allocation functions for libEnsemble, as well as example calling scripts. -Additional examples from libEnsemble's users are available in -the `libEnsemble Community Repository`_, with corresponding generator documentation available -:doc:`here`. +The examples come from the libEnsemble repository and the `libEnsemble Community Repository`_. .. toctree:: :maxdepth: 2 diff --git a/docs/examples/gen_funcs.rst b/docs/examples/gen_funcs.rst index 8e5af973b1..9bdaa311d9 100644 --- a/docs/examples/gen_funcs.rst +++ b/docs/examples/gen_funcs.rst @@ -1,22 +1,124 @@ Generator Functions =================== -.. include:: ../introduction.rst - :start-after: example_packages +Here we list many generator functions included with libEnsemble. .. IMPORTANT:: - See the API for generation functions :ref:`here`. + See the API for generator functions :ref:`here`. + +Sampling +-------- + +.. toctree:: + :maxdepth: 1 + :caption: Sampling + :hidden: + + sampling + persistent_sampling + persistent_sampling_var_resources + +- :doc:`Basic sampling` + + Various generators for sampling a space. The non-persistent function is called as needed. + +- :doc:`Persistent sampling` + + Various persistent generators (persists on a worker) for sampling a space. After the initial + batch each generator creates ``p`` new random points for every ``p`` points that are returned. + +- :doc:`Persistent sampling with variable resources` + + Various persistent sampling generators that assign different resources to each simulation. + +Optimization +------------ + +.. toctree:: + :maxdepth: 1 + :caption: Optimization + :hidden: + + aposmm + uniform_or_localopt + ax_multitask + VTMOP + ytopt + consensus + DEAP-NSGA-II + +- :doc:`APOSMM` + + Asynchronously Parallel Optimization Solver for finding Multiple Minima (APOSMM_). + +- :doc:`Ax Multitask` + + Bayesian optimization with a Gaussian process driven by an Ax_ multi-task algorithm. + +- :ref:`DEAP-NSGA-II` + + Distributed evolutionary algorithms (*community example*) + +- :ref:`Distributed optimization` + + Distributed optimization methods for minimizing sums of convex functions. (*community example*) + +- :doc:`uniform_or_localopt` + + Samples uniformly in non-persistent mode then runs an NLopt_ local optimization runs in persistent mode. + +- :ref:`VTMOP` + + Multiobjective multidisciplinary design optimization using the VTMOP_ Fortran package. (*community example*) + +- :ref:`ytopt` + + Bayesian Optimization package for determining optimal input parameter configurations for applications/executables using ytopt_. (*community example*) + +Modeling and Approximation +-------------------------- .. toctree:: - :maxdepth: 2 - :caption: Documented Example Generator Functions: + :maxdepth: 1 + :caption: Modeling and Approximation + :hidden: - sampling - aposmm - uniform_or_localopt - tasmanian - fd_param_finder - surmise - ax_multitask + gpcam + tasmanian + fd_param_finder + surmise + +- :doc:`Finite-difference parameter finder` + + Uses ECNoise_ to determine a suitable finite difference parameters for a mapping ``F`` from ``R^n`` to ``R^m``. + +- :doc:`gpCAM` + + Gaussian Process-based adaptive sampling using gpcam_. + +- :doc:`surmise` + + Modular Bayesian calibration/inference framework using Surmise_ (demonstration of cancelling previous issued simulations). + +- :doc:`Tasmanian` + + Evaluates points generators by the Tasmanian_ sparse grid library .. _libEnsemble Community Repository: https://github.com/Libensemble/libe-community-examples + +.. _APOSMM: https://link.springer.com/article/10.1007/s12532-017-0131-4 +.. _Ax: https://github.com/facebook/Ax +.. _Community Examples repository: https://github.com/Libensemble/libe-community-examples +.. _DEAP: https://deap.readthedocs.io/en/master/overview.html +.. _DFO-LS: https://github.com/numericalalgorithmsgroup/dfols +.. _ECNoise: https://www.mcs.anl.gov/~wild/cnoise/ +.. _gpcam: https://gpcam.lbl.gov/ +.. _IPAC manuscript: https://doi.org/10.18429/JACoW-ICAP2018-SAPAF03 +.. _NLopt: https://nlopt.readthedocs.io/en/latest/ +.. _OPAL: http://amas.web.psi.ch/docs/opal/opal_user_guide-1.6.0.pdf +.. _Surmise: https://surmise.readthedocs.io/en/latest/index.html +.. _Tasmanian: https://github.com/ORNL/Tasmanian +.. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html +.. _VTMOP: https://github.com/Libensemble/libe-community-examples#vtmop +.. _WarpX: https://warpx.readthedocs.io/en/latest/ +.. _ytopt: https://github.com/ytopt-team/ytopt diff --git a/docs/examples/gpcam.rst b/docs/examples/gpcam.rst new file mode 100644 index 0000000000..48a5a97628 --- /dev/null +++ b/docs/examples/gpcam.rst @@ -0,0 +1,14 @@ +gpCAM +----- + +Required: gpcam_ + +Example usage: test_gpCAM_ + +This module provides persistent generators for Gaussian Process-based adaptive sampling. + +.. automodule:: persistent_gpCAM + :members: + +.. _gpcam: https://pypi.org/project/gpcam +.. _test_gpCAM: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/regression_tests/test_gpCAM.py diff --git a/docs/examples/persistent_sampling.rst b/docs/examples/persistent_sampling.rst new file mode 100644 index 0000000000..7f778a8e8c --- /dev/null +++ b/docs/examples/persistent_sampling.rst @@ -0,0 +1,12 @@ +persistent_sampling +------------------- + +.. automodule:: persistent_sampling + :members: + :undoc-members: + +.. dropdown:: :underline:`persistent_sampling.py` + + .. literalinclude:: ../../libensemble/gen_funcs/persistent_sampling.py + :language: python + :linenos: diff --git a/docs/examples/persistent_sampling_var_resources.rst b/docs/examples/persistent_sampling_var_resources.rst new file mode 100644 index 0000000000..d4f7f8b5c5 --- /dev/null +++ b/docs/examples/persistent_sampling_var_resources.rst @@ -0,0 +1,6 @@ +persistent_sampling_var_resources +--------------------------------- + +.. automodule:: persistent_sampling_var_resources + :members: + :undoc-members: diff --git a/docs/examples/sampling.rst b/docs/examples/sampling.rst index d16753aab3..4e51b9a944 100644 --- a/docs/examples/sampling.rst +++ b/docs/examples/sampling.rst @@ -13,21 +13,3 @@ sampling .. literalinclude:: ../../libensemble/gen_funcs/sampling.py :language: python :linenos: - -persistent_sampling ---------------------------- -.. automodule:: persistent_sampling - :members: - :undoc-members: - -.. dropdown:: :underline:`persistent_sampling.py` - - .. literalinclude:: ../../libensemble/gen_funcs/persistent_sampling.py - :language: python - :linenos: - -persistent_sampling_var_resources ---------------------------------- -.. automodule:: persistent_sampling_var_resources - :members: - :undoc-members: diff --git a/docs/examples/sim_funcs.rst b/docs/examples/sim_funcs.rst index 81b922d366..be4374d884 100644 --- a/docs/examples/sim_funcs.rst +++ b/docs/examples/sim_funcs.rst @@ -13,55 +13,52 @@ function launching tasks, see the .. role:: underline :class: underline -six_hump_camel --------------- -.. automodule:: six_hump_camel - :members: - :undoc-members: +Simple simulation functions +--------------------------- +Ideal for simple debugging of generator processes or system testing. -.. dropdown:: :underline:`six_hump_camel.py` +.. toctree:: + :maxdepth: 1 - .. literalinclude:: ../../libensemble/sim_funcs/six_hump_camel.py - :language: python - :linenos: + Borehole function + Borehole function with kills + Chwirut1 vector-valued function + Inverse Bayesian likelihood + Norm + Rosenbrock test optimization function + Six Hump Camel + Test noisy function + Test periodic function -chwirut -------- -.. automodule:: chwirut1 - :members: - :undoc-members: +Functions that run user applications +------------------------------------ -noisy_vector_mapping --------------------- -.. automodule:: noisy_vector_mapping - :members: - :undoc-members: +These use the executor to launch applications and in some cases +handle dynamic CPU/GPU allocation. -.. dropdown:: :underline:`noisy_vector_mapping.py` +The ``Variable resources`` module contains basic examples, while the ``Template`` +examples use a simple MPI/OpenMP (with GPU offload option) application (``forces``) +to demonstrate libEnsemble’s capabilities on various HPC systems. The +build_forces.sh_ file gives compile lines for building the simple ``forces`` +application on various platforms (use -DGPU to build for GPU). - .. literalinclude:: ../../libensemble/sim_funcs/noisy_vector_mapping.py - :language: python - :linenos: +.. toctree:: + :maxdepth: 1 -periodic_func -------------- -.. automodule:: periodic_func - :members: - :undoc-members: + Variable resources + sim_funcs/forces_simf + sim_funcs/forces_simf_input_file + sim_funcs/forces_simf_gpu + sim_funcs/forces_simf_gpu_vary_resources + sim_funcs/forces_simf_gpu_multi_app + WarpX Example -borehole --------- -.. automodule:: borehole - :members: - :undoc-members: +Special simulation functions +---------------------------- -executor_hworld ---------------- -.. automodule:: executor_hworld - :members: +.. toctree:: + :maxdepth: 1 -.. dropdown:: :underline:`executor_hworld.py` + sim_funcs/mock_sim - .. literalinclude:: ../../libensemble/sim_funcs/executor_hworld.py - :language: python - :linenos: +.. _build_forces.sh: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/scaling_tests/forces/forces_app/build_forces.sh diff --git a/docs/examples/sim_funcs/borehole.rst b/docs/examples/sim_funcs/borehole.rst new file mode 100644 index 0000000000..88e16ed9c7 --- /dev/null +++ b/docs/examples/sim_funcs/borehole.rst @@ -0,0 +1,14 @@ +borehole +-------------- +.. role:: underline + :class: underline + +.. automodule:: borehole + :members: + :undoc-members: + +.. dropdown:: :underline:`borehole.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/borehole.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/borehole_kills.rst b/docs/examples/sim_funcs/borehole_kills.rst new file mode 100644 index 0000000000..fb3f06381e --- /dev/null +++ b/docs/examples/sim_funcs/borehole_kills.rst @@ -0,0 +1,14 @@ +borehole_kills +-------------- +.. role:: underline + :class: underline + +.. automodule:: borehole_kills + :members: + :undoc-members: + +.. dropdown:: :underline:`borehole_kills.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/borehole_kills.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/chwirut1.rst b/docs/examples/sim_funcs/chwirut1.rst new file mode 100644 index 0000000000..2112abbdf0 --- /dev/null +++ b/docs/examples/sim_funcs/chwirut1.rst @@ -0,0 +1,14 @@ +chwirut1 +-------------- +.. role:: underline + :class: underline + +.. automodule:: chwirut1 + :members: + :undoc-members: + +.. dropdown:: :underline:`chwirut1.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/chwirut1.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/forces_simf.rst b/docs/examples/sim_funcs/forces_simf.rst new file mode 100644 index 0000000000..2aa3c2664c --- /dev/null +++ b/docs/examples/sim_funcs/forces_simf.rst @@ -0,0 +1,23 @@ +Template for CPU executables +---------------------------- + +.. role:: underline + :class: underline + +.. automodule:: forces_simple.forces_simf + :members: + :undoc-members: + +.. dropdown:: :underline:`forces_simf.py` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_simple/forces_simf.py + :language: python + :linenos: + +.. dropdown:: :underline:`Example usage` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_simple/run_libe_forces.py + :language: python + :linenos: + +Also see the :doc:`Forces tutorial <../../tutorials/executor_forces_tutorial>`. diff --git a/docs/examples/sim_funcs/forces_simf_gpu.rst b/docs/examples/sim_funcs/forces_simf_gpu.rst new file mode 100644 index 0000000000..4c74d254f1 --- /dev/null +++ b/docs/examples/sim_funcs/forces_simf_gpu.rst @@ -0,0 +1,27 @@ +Template for GPU executables +---------------------------- + +.. role:: underline + :class: underline + +.. automodule:: forces_gpu.forces_simf + :members: + :undoc-members: + +.. dropdown:: :underline:`forces_simf.py` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_gpu/forces_simf.py + :language: python + :linenos: + +.. dropdown:: :underline:`Example usage` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_gpu/run_libe_forces.py + :language: python + :linenos: + +Also see the :doc:`Forces GPU tutorial <../../tutorials/forces_gpu_tutorial>` and the +video_ demonstration on Frontier_. + +.. _video: https://www.youtube.com/watch?v=H2fmbZ6DnVc +.. _Frontier: https://docs.olcf.ornl.gov/systems/frontier_user_guide.html diff --git a/docs/examples/sim_funcs/forces_simf_gpu_multi_app.rst b/docs/examples/sim_funcs/forces_simf_gpu_multi_app.rst new file mode 100644 index 0000000000..8a491e8816 --- /dev/null +++ b/docs/examples/sim_funcs/forces_simf_gpu_multi_app.rst @@ -0,0 +1,44 @@ +Template with dynamic resources (CPU app and GPU app) +----------------------------------------------------- + +.. role:: underline + :class: underline + +Launches either the CPU-only or GPU version of the forces MPI app and auto-assigns +ranks and GPU resources as requested by the generator. + +This makes efficient use of each node as the expensive GPU simulations will use the GPUs on +the node/s, while the rest of the CPU cores are assigned to the simple CPU-only simulations. + +See this publication_ for a real-world demonstration of these capabilities. + +.. automodule:: forces_multi_app.forces_simf + :members: + :undoc-members: + +.. dropdown:: :underline:`forces_simf.py` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_multi_app/forces_simf.py + :language: python + :linenos: + +The generator in the example below assigns to each simulation either the CPU application +or the GPU application and also randomly assigns the number of processors for each +simulation. For the GPU application, one GPU is used for each MPI rank. As many nodes +as necessary will be used for each application. + +The special generator output fields "num_procs" and "num_gpus" are automatically picked +up by each worker and these will be used when the simulation is run, unless overridden. + +.. dropdown:: :underline:`Example usage` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_multi_app/run_libe_forces.py + :language: python + :linenos: + +More information is available in the :doc:`Forces GPU tutorial <../../tutorials/forces_gpu_tutorial>` +and the video_ demonstration on Frontier_. + +.. _Frontier: https://docs.olcf.ornl.gov/systems/frontier_user_guide.html +.. _publication: https://doi.org/10.1103/PhysRevAccelBeams.26.084601 +.. _video: https://www.youtube.com/watch?v=H2fmbZ6DnVc diff --git a/docs/examples/sim_funcs/forces_simf_gpu_vary_resources.rst b/docs/examples/sim_funcs/forces_simf_gpu_vary_resources.rst new file mode 100644 index 0000000000..da87171b12 --- /dev/null +++ b/docs/examples/sim_funcs/forces_simf_gpu_vary_resources.rst @@ -0,0 +1,32 @@ +Template for GPU executables with dynamic resources +--------------------------------------------------- + +.. role:: underline + :class: underline + +.. automodule:: forces_gpu_var_resources.forces_simf + :members: + :undoc-members: + +.. dropdown:: :underline:`forces_simf.py` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_gpu_var_resources/forces_simf.py + :language: python + :linenos: + +.. dropdown:: :underline:`Example usage` + + Note the use of the generator function ``uniform_sample_with_var_gpus`` that sets ``num_gpus`` as a ``gen_specs`` output field corresponding to each generated simulation input. + + The special generator output field "num_gpus" is automatically picked up by each worker + and will be used when the simulation is run, unless overridden. + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_gpu_var_resources/run_libe_forces.py + :language: python + :linenos: + +Also see the :doc:`Forces GPU tutorial <../../tutorials/forces_gpu_tutorial>` and the +video_ demonstration on Frontier_. + +.. _video: https://www.youtube.com/watch?v=H2fmbZ6DnVc +.. _Frontier: https://docs.olcf.ornl.gov/systems/frontier_user_guide.html diff --git a/docs/examples/sim_funcs/forces_simf_input_file.rst b/docs/examples/sim_funcs/forces_simf_input_file.rst new file mode 100644 index 0000000000..bc304f8815 --- /dev/null +++ b/docs/examples/sim_funcs/forces_simf_input_file.rst @@ -0,0 +1,43 @@ +Template for CPU executables with input file +-------------------------------------------- + +.. role:: underline + :class: underline + +Many applications read an input file instead of being given parameters directly +on the run line. + +In this variant of the forces example, a templated input file is parameterized +for each evaluation. + +This requires **jinja2** to be installed:: + + pip install jinja2 + +In the example, the file ``forces_input`` contains the following (remember +we are using particles as seed also for simplicity):: + + num_particles = {{particles}} + num_steps = 10 + rand_seed = {{particles}} + +libEnsemble will copy this input file to each simulation directory. There, the +simulation function will updates the input file with the ``particles`` value for +this simulation. + +.. automodule:: forces_simple_with_input_file.forces_simf + :members: run_forces + +.. dropdown:: :underline:`forces_simf.py` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_simple_with_input_file/forces_simf.py + :language: python + :linenos: + +.. dropdown:: :underline:`Example usage` + + .. literalinclude:: ../../../libensemble/tests/scaling_tests/forces/forces_simple_with_input_file/run_libe_forces.py + :language: python + :linenos: + +Also see the :doc:`Forces tutorial <../../tutorials/executor_forces_tutorial>`. diff --git a/docs/examples/sim_funcs/inverse_bayes.rst b/docs/examples/sim_funcs/inverse_bayes.rst new file mode 100644 index 0000000000..4320fb0be8 --- /dev/null +++ b/docs/examples/sim_funcs/inverse_bayes.rst @@ -0,0 +1,14 @@ +inverse_bayes +-------------- +.. role:: underline + :class: underline + +.. automodule:: inverse_bayes + :members: + :undoc-members: + +.. dropdown:: :underline:`inverse_bayes.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/inverse_bayes.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/mock_sim.rst b/docs/examples/sim_funcs/mock_sim.rst new file mode 100644 index 0000000000..4555ab0b6c --- /dev/null +++ b/docs/examples/sim_funcs/mock_sim.rst @@ -0,0 +1,24 @@ +Mock simulation evaluations (re-run using history file) +------------------------------------------------------- + +.. role:: underline + :class: underline + +.. automodule:: mock_sim + :members: + :undoc-members: + +.. dropdown:: :underline:`mock_sim.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/mock_sim.py + :language: python + :linenos: + +.. dropdown:: :underline:`Example usage` + + This test runs two repetitions. The first ensemble dumps a history file, the second replays + the first run using the mock sim with the history file. + + .. literalinclude:: ../../../libensemble/tests/functionality_tests/test_uniform_sampling.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/noisy_vector_mapping.rst b/docs/examples/sim_funcs/noisy_vector_mapping.rst new file mode 100644 index 0000000000..4b0eab8878 --- /dev/null +++ b/docs/examples/sim_funcs/noisy_vector_mapping.rst @@ -0,0 +1,15 @@ +noisy_vector_mapping +-------------------- + +.. role:: underline + :class: underline + +.. automodule:: noisy_vector_mapping + :members: + :undoc-members: + +.. dropdown:: :underline:`noisy_vector_mapping.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/noisy_vector_mapping.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/periodic_func.rst b/docs/examples/sim_funcs/periodic_func.rst new file mode 100644 index 0000000000..42e6298b87 --- /dev/null +++ b/docs/examples/sim_funcs/periodic_func.rst @@ -0,0 +1,14 @@ +periodic_func +-------------- +.. role:: underline + :class: underline + +.. automodule:: periodic_func + :members: + :undoc-members: + +.. dropdown:: :underline:`periodic_func.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/periodic_func.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/rosenbrock.rst b/docs/examples/sim_funcs/rosenbrock.rst new file mode 100644 index 0000000000..6eceae2034 --- /dev/null +++ b/docs/examples/sim_funcs/rosenbrock.rst @@ -0,0 +1,14 @@ +rosenbrock +---------- + +.. role:: underline + :class: underline + +.. automodule:: rosenbrock + :members: + +.. dropdown:: :underline:`rosenbrock.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/rosenbrock.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/simple_sim.rst b/docs/examples/sim_funcs/simple_sim.rst new file mode 100644 index 0000000000..ae9fdc8380 --- /dev/null +++ b/docs/examples/sim_funcs/simple_sim.rst @@ -0,0 +1,14 @@ +simple_sim +-------------- +.. role:: underline + :class: underline + +.. automodule:: simple_sim + :members: + :undoc-members: + +.. dropdown:: :underline:`simple_sim.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/simple_sim.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/six_hump_camel.rst b/docs/examples/sim_funcs/six_hump_camel.rst new file mode 100644 index 0000000000..aca6fccae2 --- /dev/null +++ b/docs/examples/sim_funcs/six_hump_camel.rst @@ -0,0 +1,14 @@ +six_hump_camel +-------------- +.. role:: underline + :class: underline + +.. automodule:: six_hump_camel + :members: + :undoc-members: + +.. dropdown:: :underline:`six_hump_camel.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/six_hump_camel.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/surmise_test_function.rst b/docs/examples/sim_funcs/surmise_test_function.rst new file mode 100644 index 0000000000..97a871f5d2 --- /dev/null +++ b/docs/examples/sim_funcs/surmise_test_function.rst @@ -0,0 +1,15 @@ +surmise_test_function +--------------------- + +.. role:: underline + :class: underline + +.. automodule:: surmise_test_function + :members: + :undoc-members: + +.. dropdown:: :underline:`surmise_test_function.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/surmise_test_function.py + :language: python + :linenos: diff --git a/docs/examples/sim_funcs/var_resources.rst b/docs/examples/sim_funcs/var_resources.rst new file mode 100644 index 0000000000..713fbcf6bb --- /dev/null +++ b/docs/examples/sim_funcs/var_resources.rst @@ -0,0 +1,14 @@ +var_resources +-------------- +.. role:: underline + :class: underline + +.. automodule:: var_resources + :members: + :undoc-members: + +.. dropdown:: :underline:`var_resources.py` + + .. literalinclude:: ../../../libensemble/sim_funcs/var_resources.py + :language: python + :linenos: diff --git a/docs/examples/submission_scripts.rst b/docs/examples/submission_scripts.rst new file mode 100644 index 0000000000..7c37da40ea --- /dev/null +++ b/docs/examples/submission_scripts.rst @@ -0,0 +1,5 @@ +.. include:: ../platforms/example_scripts.rst + :end-before: .. _slurm_mpi_distributed: + +.. include:: ../platforms/example_scripts.rst + :start-after: .. _slurm_mpi_distributed: diff --git a/docs/examples/surmise.rst b/docs/examples/surmise.rst index 69d0f068ec..f38b9b79a2 100644 --- a/docs/examples/surmise.rst +++ b/docs/examples/surmise.rst @@ -1,7 +1,12 @@ persistent_surmise ------------------ -Required: Surmise_ +Required: Surmise_. + +Example usage: basic_, `kill canceled sims`_ + +Modular Bayesian calibration/inference framework. + The :doc:`Borehole Calibration tutorial<../tutorials/calib_cancel_tutorial>` uses this generator as an example of the capability to cancel pending simulations. @@ -10,3 +15,5 @@ example of the capability to cancel pending simulations. :no-undoc-members: .. _Surmise: https://github.com/bandframework/surmise +.. _basic: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/regression_tests/test_persistent_surmise_calib.py +.. _`kill canceled sims`: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py diff --git a/docs/examples/tasmanian.rst b/docs/examples/tasmanian.rst index 9ddd8b054a..0099616f32 100644 --- a/docs/examples/tasmanian.rst +++ b/docs/examples/tasmanian.rst @@ -3,13 +3,15 @@ persistent_tasmanian Required: Tasmanian_, pypackaging_, scikit-build_ +Example usage: batched_, async_ + Note that Tasmanian can be pip installed, but currently must use either *venv* or *--user* install. ``E.g: pip install scikit-build packaging Tasmanian --user`` .. automodule:: persistent_tasmanian - :members: + :members: sparse_grid_batched, sparse_grid_async :undoc-members: .. role:: underline @@ -23,4 +25,7 @@ use either *venv* or *--user* install. .. _pypackaging: https://pypi.org/project/pypackaging/ .. _scikit-build: https://scikit-build.readthedocs.io/en/latest/index.html -.. _Tasmanian: https://tasmanian.ornl.gov/ + +.. Caution - tasmanian_ is named in example docstring so must be named differently +.. _batched: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/regression_tests/test_persistent_tasmanian.py +.. _async: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/regression_tests/test_persistent_tasmanian_async.py diff --git a/docs/images/centralized_new_detailed.png b/docs/images/centralized_dedicated.png similarity index 100% rename from docs/images/centralized_new_detailed.png rename to docs/images/centralized_dedicated.png diff --git a/docs/images/centralized_gen_on_manager.png b/docs/images/centralized_gen_on_manager.png new file mode 100644 index 0000000000..949f8e3968 Binary files /dev/null and b/docs/images/centralized_gen_on_manager.png differ diff --git a/docs/images/gpcam_model_improvement.png b/docs/images/gpcam_model_improvement.png new file mode 100644 index 0000000000..232469be2f Binary files /dev/null and b/docs/images/gpcam_model_improvement.png differ diff --git a/docs/index.rst b/docs/index.rst index 3cfa749e97..9f7093ff10 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ tutorials/local_sine_tutorial tutorials/executor_forces_tutorial tutorials/forces_gpu_tutorial + tutorials/gpcam_tutorial tutorials/aposmm_tutorial tutorials/calib_cancel_tutorial @@ -34,6 +35,7 @@ examples/sim_funcs examples/alloc_funcs examples/calling_scripts + Submission Scripts .. toctree:: :maxdepth: 1 diff --git a/docs/introduction.rst b/docs/introduction.rst index bbea8d9d43..4b36943398 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -8,6 +8,7 @@ See the `user guide`_ for more information. .. example_packages .. dropdown:: **Example Compatible Packages** + :open: libEnsemble and the `Community Examples repository`_ include example generator functions for the following libraries: @@ -49,7 +50,7 @@ See the `user guide`_ for more information. .. _PETSc/TAO: http://www.mcs.anl.gov/petsc .. _scipy.optimize: https://docs.scipy.org/doc/scipy/reference/optimize.html .. _Surmise: https://surmise.readthedocs.io/en/latest/index.html -.. _Tasmanian: https://tasmanian.ornl.gov/ +.. _Tasmanian: https://github.com/ORNL/Tasmanian .. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html .. _VTMOP: https://github.com/Libensemble/libe-community-examples#vtmop .. _WarpX: https://warpx.readthedocs.io/en/latest/ diff --git a/docs/introduction_latex.rst b/docs/introduction_latex.rst index d271c085bb..e7750bac5f 100644 --- a/docs/introduction_latex.rst +++ b/docs/introduction_latex.rst @@ -50,7 +50,7 @@ .. _Surmise: https://surmise.readthedocs.io/en/latest/index.html .. _SWIG: http://swig.org/ .. _tarball: https://github.com/Libensemble/libensemble/releases/latest -.. _Tasmanian: https://tasmanian.ornl.gov/ +.. _Tasmanian: https://github.com/ORNL/Tasmanian .. _tomli: https://pypi.org/project/tomli/ .. _tqdm: https://tqdm.github.io/ .. _user guide: https://libensemble.readthedocs.io/en/latest/programming_libE.html diff --git a/docs/nitpicky b/docs/nitpicky index 8315471a11..e43a0760bb 100644 --- a/docs/nitpicky +++ b/docs/nitpicky @@ -37,19 +37,17 @@ py:class py:class +ScalarType py:class +_ScalarType_co py:class collections.abc.Sequence +py:class pathlib._local.Path # Internal paths that are verified importable but Sphinx can't find py:class libensemble.resources.platforms.Aurora py:class libensemble.resources.platforms.GenericROCm -py:class libensemble.resources.platforms.Crusher py:class libensemble.resources.platforms.Frontier py:class libensemble.resources.platforms.Perlmutter py:class libensemble.resources.platforms.PerlmutterCPU py:class libensemble.resources.platforms.PerlmutterGPU py:class libensemble.resources.platforms.Polaris -py:class libensemble.resources.platforms.Spock py:class libensemble.resources.platforms.Summit -py:class libensemble.resources.platforms.Sunspot py:class libensemble.resources.rset_resources.RSetResources py:class libensemble.resources.env_resources.EnvResources py:class libensemble.resources.resources.Resources diff --git a/docs/platforms/aurora.rst b/docs/platforms/aurora.rst index 864d5bedf1..af2e7cc160 100644 --- a/docs/platforms/aurora.rst +++ b/docs/platforms/aurora.rst @@ -12,10 +12,16 @@ nodes. Configuring Python and Installation ----------------------------------- -To obtain Python use:: +To obtain Python and create a virtual environment: + +.. code-block:: console - module use /soft/modulefiles module load frameworks + python -m venv /path/to-venv --system-site-packages + . /path/to-venv/bin/activate + +where ``/path/to-venv`` can be anywhere you have write access. For future sessions, +just load the frameworks module and run the activate line. To obtain libEnsemble:: @@ -31,7 +37,7 @@ To run the :doc:`forces_gpu<../tutorials/forces_gpu_tutorial>` tutorial on Aurora. To obtain the example you can git clone libEnsemble - although only -the forces sub-directory is needed:: +the ``forces`` sub-directory is strictly needed:: git clone https://github.com/Libensemble/libensemble cd libensemble/libensemble/tests/scaling_tests/forces/forces_app @@ -44,9 +50,11 @@ Now go to forces_gpu directory:: cd ../forces_gpu -To make use of all available GPUs, open ``run_libe_forces.py`` and adjust -the exit_criteria to do more simulations. The following will do two -simulations for each worker:: +To make use of all available GPUs, open **run_libe_forces.py** and adjust +the ``exit_criteria`` to perform more simulations. The following will run two +simulations for each worker: + +.. code-block:: python # Instruct libEnsemble to exit after this many simulations ensemble.exit_criteria = ExitCriteria(sim_max=nsim_workers*2) @@ -54,17 +62,16 @@ simulations for each worker:: Now grab an interactive session on two nodes (or use the batch script at ``../submission_scripts/submit_pbs_aurora.sh``):: - qsub -A -l select=2 -l walltime=15:00 -lfilesystems=home -q EarlyAppAccess -I + qsub -A -l select=2 -l walltime=15:00 -lfilesystems=home:flare -q debug -I Once in the interactive session, you may need to reload the frameworks module:: cd $PBS_O_WORKDIR - module use /soft/modulefiles - module load frameworks + . /path/to-venv/bin/activate Then in the session run:: - python run_libe_forces.py --comms local --nworkers 13 + python run_libe_forces.py -n 13 This provides twelve workers for running simulations (one for each GPU across two nodes). An extra worker is added to run the persistent generator. The @@ -72,12 +79,28 @@ GPU settings for each worker simulation are printed. Looking at ``libE_stats.txt`` will provide a summary of the runs. +Now try running:: + + ./cleanup.sh + python run_libe_forces.py -n 7 + +And you will see it runs with two cores and two GPUs are used per +worker. The **forces** example automatically uses the GPUs available to +each worker. + +Live viewing GPU usage +---------------------- + +To see GPU usage, SSH into a compute node you are on in another window and run:: + + module load xpu-smi + watch -n 0.1 xpu-smi dump -d -1 -m 0 -n 1 + Using tiles as GPUs ------------------- -If you wish to treat each tile as its own GPU, then add the *libE_specs* -option ``use_tiles_as_gpus=True``, so the *libE_specs* block of -``run_libe_forces.py`` becomes: +To treat each tile as its own GPU, add the ``use_tiles_as_gpus=True`` option +to the ``libE_specs`` block in **run_libe_forces.py**: .. code-block:: python @@ -90,19 +113,43 @@ option ``use_tiles_as_gpus=True``, so the *libE_specs* block of Now you can run again but with twice the workers for running simulations (each will use one GPU tile):: - python run_libe_forces.py --comms local --nworkers 25 + python run_libe_forces.py -n 25 + +Running generator on the manager +-------------------------------- + +An alternative is to run the generator on a thread on the manager. The +number of workers can then be set to the number of simulation workers. + +Change the ``libE_specs`` in **run_libe_forces.py** as follows: + +.. code-block:: python + + nsim_workers = ensemble.nworkers + + # Persistent gen does not need resources + ensemble.libE_specs = LibeSpecs( + gen_on_manager=True, + +then we can run with 12 (instead of 13) workers:: + + python run_libe_forces.py -n 12 + +Dynamic resource assignment +--------------------------- -Note that the *forces* example will automatically use the GPUs available to -each worker (with one MPI rank per GPU), so if fewer workers are provided, -more than one GPU will be used per simulation. +In the **forces** directory you will also find: -Also see ``forces_gpu_var_resources`` and ``forces_multi_app`` examples for -cases that use varying processor/GPU counts per simulation. +* ``forces_gpu_var_resources`` uses varying processor/GPU counts per simulation. +* ``forces_multi_app`` uses varying processor/GPU counts per simulation and also + uses two different user executables, one which is CPU-only and one which + uses GPUs. This allows highly efficient use of nodes for multi-application + ensembles. Demonstration ------------- -Note that a video demonstration_ of the *forces_gpu* example on *Frontier* +Note that a video demonstration_ of the *forces_gpu* example on **Frontier** is also available. The workflow is identical when running on Aurora, with the exception of different compiler options and numbers of workers (because the numbers of GPUs on a node differs). diff --git a/docs/platforms/bebop.rst b/docs/platforms/bebop.rst index 80f4a56024..e57172c1b3 100644 --- a/docs/platforms/bebop.rst +++ b/docs/platforms/bebop.rst @@ -2,7 +2,7 @@ Bebop ===== -Bebop_ is a Cray CS400 cluster with Intel Broadwell and Knights Landing compute +Bebop_ is a Cray CS400 cluster with Intel Broadwell compute nodes available in the Laboratory Computing Resources Center (LCRC) at Argonne National Laboratory. @@ -52,24 +52,24 @@ for installing libEnsemble. Job Submission -------------- -Bebop uses Slurm_ for job submission and management. The two commands you'll -likely use the most to run jobs are ``srun`` and ``sbatch`` for running -interactively and batch, respectively. - -libEnsemble node-worker affinity is especially flexible on Bebop. By adjusting -``srun`` runtime options_ users may assign multiple libEnsemble workers to each -allocated node(oversubscription) or assign multiple nodes per worker. +Bebop uses PBS for job submission and management. Interactive Runs ^^^^^^^^^^^^^^^^ -You can allocate four Knights Landing nodes for thirty minutes through the following:: +You can allocate four Broadwell nodes for thirty minutes through the following:: + + qsub -I -A -l select=4:mpiprocs=4 -l walltime=30:00 - salloc -N 4 -p knl -A [username OR project] -t 00:30:00 +Once in the interactive session, you may need to reload your modules:: -With your nodes allocated, queue your job to start with four MPI ranks:: + cd $PBS_O_WORKDIR + module load anaconda3 gcc openmpi aocl + conda activate bebop_libe_env - srun -n 4 python calling.py +Now run your script with four workers (one for generator and three for simulations):: + + python my_libe_script.py --nworkers 4 ``mpirun`` should also work. This line launches libEnsemble with a manager and **three** workers to one allocated compute node, with three nodes available for @@ -83,57 +83,10 @@ be initiated with ``libE_specs["dedicated_mode"]=True`` and not oversubscribing, specify one more MPI process than the number of allocated nodes. The manager and first worker run together on a node. -If you would like to interact directly with the compute nodes via a shell, -the following starts a bash session on a Knights Landing node -for thirty minutes:: - - srun --pty -A [username OR project] -p knl -t 00:30:00 /bin/bash - .. note:: You will need to reactivate your conda virtual environment and reload your modules! Configuring this routine to occur automatically is recommended. -Batch Runs -^^^^^^^^^^ - -Batch scripts specify run settings using ``#SBATCH`` statements. A simple example -for a libEnsemble use case running in :doc:`distributed` MPI -mode on Broadwell nodes resembles the following: - -.. code-block:: bash - :linenos: - - #!/bin/bash - #SBATCH -J myjob - #SBATCH -N 4 - #SBATCH -p bdwall - #SBATCH -A myproject - #SBATCH -o myjob.out - #SBATCH -e myjob.error - #SBATCH -t 00:15:00 - - # These four lines construct a machinefile for the executor and slurm - srun hostname | sort -u > node_list - head -n 1 node_list > machinefile.$SLURM_JOBID - cat node_list >> machinefile.$SLURM_JOBID - export SLURM_HOSTFILE=machinefile.$SLURM_JOBID - - srun --ntasks 5 python calling_script.py - -With this saved as ``myscript.sh``, allocating, configuring, and running libEnsemble -on Bebop is achieved by running :: - - sbatch myscript.sh - -Example submission scripts for running on Bebop in distributed and centralized mode -are also given in the :doc:`examples`. - -Debugging Strategies --------------------- - -View the status of your submitted jobs with ``squeue``, and cancel jobs with -``scancel ``. - Additional Information ---------------------- @@ -144,5 +97,3 @@ See the LCRC Bebop docs here_ for more information about Bebop. .. _conda: https://conda.io/en/latest/ .. _here: https://docs.lcrc.anl.gov/bebop/running-jobs-bebop/ .. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ -.. _options: https://slurm.schedmd.com/srun.html -.. _Slurm: https://slurm.schedmd.com/ diff --git a/docs/platforms/example_scripts.rst b/docs/platforms/example_scripts.rst index 90db3a63a6..d534f0c662 100644 --- a/docs/platforms/example_scripts.rst +++ b/docs/platforms/example_scripts.rst @@ -2,41 +2,103 @@ Example Scheduler Submission Scripts ==================================== Below are example submission scripts used to configure and launch libEnsemble -on a variety of high-powered systems. See :doc:`here` for more -information about the respective systems and configuration. +on a variety of high-powered systems. See :ref:`Running on HPC Systems` +for more information about the respective systems and configuration. -.. dropdown:: Slurm - Basic +.. note:: + It is **highly recommended** that the directive lines (e.g., #SBATCH) in batch + submission scripts do **NOT** specify processor, task, or GPU configuration + information---these lines should only specify the number of nodes required. - .. literalinclude:: ../../examples/libE_submission_scripts/submit_slurm_simple.sh - :caption: /examples/libE_submission_scripts/submit_slurm_simple.sh - :language: bash + For example, do not specify ``#SBATCH --gpus-per-node=4`` in order to use four + GPUs on the node, when each worker may use less than this, as this may assign + all of the GPUs to a single MPI invocation. Instead, the configuration should + be supplied either + :doc:`in the simulation function<../examples/sim_funcs/forces_simf_gpu>` + or, if using dynamic resources, + :doc:`in the generator<../examples/sim_funcs/forces_simf_gpu_vary_resources>`. -.. dropdown:: Bridges - Central Mode +General examples +---------------- - .. literalinclude:: ../../examples/libE_submission_scripts/bridges_submit_slurm_central.sh - :caption: /examples/libE_submission_scripts/bridges_submit_slurm_central.sh - :language: bash +Slurm - Basic +^^^^^^^^^^^^^ -.. dropdown:: Bebop - Central Mode +.. literalinclude:: ../../examples/libE_submission_scripts/submit_slurm_simple.sh + :caption: /examples/libE_submission_scripts/submit_slurm_simple.sh + :language: bash - .. literalinclude:: ../../examples/libE_submission_scripts/bebop_submit_slurm_central.sh - :caption: /examples/libE_submission_scripts/bebop_submit_slurm_central.sh - :language: bash +PBS - Basic +^^^^^^^^^^^ -.. dropdown:: Bebop - Distributed Mode +.. literalinclude:: ../../examples/libE_submission_scripts/submit_pbs_simple.sh + :caption: /examples/libE_submission_scripts/submit_pbs_simple.sh + :language: bash - .. literalinclude:: ../../examples/libE_submission_scripts/bebop_submit_slurm_distrib.sh - :caption: /examples/libE_submission_scripts/bebop_submit_slurm_distrib.sh - :language: bash +LSF - Basic +^^^^^^^^^^^ -.. dropdown:: Summit - On Launch Nodes with Multiprocessing +.. literalinclude:: ../../examples/libE_submission_scripts/submit_lsf_simple.sh + :caption: /examples/libE_submission_scripts/submit_lsf_simple.sh + :language: bash - .. literalinclude:: ../../examples/libE_submission_scripts/summit_submit_mproc.sh - :caption: /examples/libE_submission_scripts/summit_submit_mproc.sh - :language: bash +System Examples +--------------- -.. dropdown:: Cobalt - Intermediate node with Multiprocessing +Aurora +^^^^^^ - .. literalinclude:: ../../examples/libE_submission_scripts/cobalt_submit_mproc.sh - :caption: /examples/libE_submission_scripts/cobalt_submit_mproc.sh - :language: bash +.. literalinclude:: ../../examples/libE_submission_scripts/submit_pbs_aurora.sh + :caption: /examples/libE_submission_scripts/submit_pbs_aurora.sh + :language: bash + +Frontier (Large WarpX Ensemble) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. literalinclude:: ../../examples/libE_submission_scripts/submit_frontier_large.sh + :caption: /examples/libE_submission_scripts/submit_frontier_large.sh + :language: bash + +Perlmutter +^^^^^^^^^^ + +.. literalinclude:: ../../examples/libE_submission_scripts/submit_perlmutter.sh + :caption: /examples/libE_submission_scripts/submit_perlmutter.sh + :language: bash + +Polaris +^^^^^^^ + +.. literalinclude:: ../../examples/libE_submission_scripts/submit_pbs_polaris.sh + :caption: /examples/libE_submission_scripts/submit_pbs_polaris.sh + :language: bash + +Bebop +^^^^^ + +.. literalinclude:: ../../examples/libE_submission_scripts/bebop_submit_pbs_central.sh + :caption: /examples/libE_submission_scripts/bebop_submit_pbs_central.sh + :language: bash + +Bridges - MPI / Central Mode +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. literalinclude:: ../../examples/libE_submission_scripts/bridges_submit_slurm_central.sh + :caption: /examples/libE_submission_scripts/bridges_submit_slurm_central.sh + :language: bash + +.. _slurm_mpi_distributed: + +SLURM - MPI / Distributed Mode (co-locate workers & MPI applications) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. literalinclude:: ../../examples/libE_submission_scripts/submit_distrib_mpi4py.sh + :caption: /examples/libE_submission_scripts/submit_distrib_mpi4py.sh + :language: bash + +Summit (Decommissioned) - On Launch Nodes with Multiprocessing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. literalinclude:: ../../examples/libE_submission_scripts/summit_submit_mproc.sh + :caption: /examples/libE_submission_scripts/summit_submit_mproc.sh + :language: bash diff --git a/docs/platforms/frontier.rst b/docs/platforms/frontier.rst index 7c7a6b18c3..4fdc7a0b36 100644 --- a/docs/platforms/frontier.rst +++ b/docs/platforms/frontier.rst @@ -64,7 +64,7 @@ Now grab an interactive session on one node:: Then in the session run:: - python run_libe_forces.py --comms local --nworkers 9 + python run_libe_forces.py --nworkers 9 This places the generator on the first worker and runs simulations on the others (each simulation using one GPU). diff --git a/docs/platforms/improv.rst b/docs/platforms/improv.rst index b108fbc46c..bdb2269a85 100644 --- a/docs/platforms/improv.rst +++ b/docs/platforms/improv.rst @@ -56,7 +56,7 @@ Once in the interactive session, you may need to reload the modules:: Now run forces with five workers (one for generator and four for simulations):: - python run_libe_forces.py --comms local --nworkers 5 + python run_libe_forces.py --nworkers 5 mpi4py comms ============ diff --git a/docs/platforms/perlmutter.rst b/docs/platforms/perlmutter.rst index c9f903dc44..88d3f808b2 100644 --- a/docs/platforms/perlmutter.rst +++ b/docs/platforms/perlmutter.rst @@ -21,7 +21,7 @@ Create a conda environment You can create a conda_ environment in which to install libEnsemble and all dependencies. For example:: - conda create -n libe-pm python=3.9 -y + conda create -n libe-pm python=3.10 -y As Perlmutter has a shared HOME filesystem with other clusters, using the ``-pm`` suffix (for Perlmutter) is good practice. @@ -125,7 +125,6 @@ and run with:: python run_libe_forces.py -n 4 - To watch video ^^^^^^^^^^^^^^ diff --git a/docs/platforms/platforms_index.rst b/docs/platforms/platforms_index.rst index 814b3e986f..c06cdbe6fd 100644 --- a/docs/platforms/platforms_index.rst +++ b/docs/platforms/platforms_index.rst @@ -3,37 +3,113 @@ Running on HPC Systems ====================== -Central vs. Distributed ------------------------ - -libEnsemble has been developed, supported, and tested on systems of highly varying -scales, from laptops to thousands of compute nodes. On multi-node systems, there are -two basic modes of configuring libEnsemble to run and launch tasks (user applications) +libEnsemble has been tested on systems of highly varying scales, from laptops to +thousands of compute nodes. On multi-node systems, there are a few alternative +ways of configuring libEnsemble to run and launch tasks (i.e., user applications) on the available nodes. -The first mode we refer to as **central** mode, where the libEnsemble manager and worker processes -are grouped onto one or more dedicated nodes. Workers launch applications onto -the remaining allocated nodes: +The :doc:`Forces tutorial <../../tutorials/executor_forces_tutorial>` gives an +example with a simple MPI application. - .. image:: ../images/centralized_new_detailed.png - :alt: centralized - :scale: 30 - :align: center +Note that while the diagrams below show one application being run per node, +configurations with **multiple nodes per worker** or **multiple workers per node** +are both common use cases. + +Centralized Running +------------------- + +The default communications scheme places the manager and workers on the first node. +The :doc:`MPI Executor<../executor/mpi_executor>` can then be invoked by each +simulation worker, and libEnsemble will distribute user applications across the +node allocation. This is the **most common approach** where each simulation +runs an MPI application. + +The generator will run on a worker by default, but if running a single generator, +the :ref:`libE_specs` option **gen_on_manager** is recommended, +which runs the generator on the manager (using a thread) as below. + +.. list-table:: + :widths: 60 40 + + * - .. image:: ../images/centralized_gen_on_manager.png + :alt: centralized + :scale: 55 + + - In calling script: + + .. code-block:: python + :linenos: + + ensemble.libE_specs = LibeSpecs( + gen_on_manager=True, + ) + + A SLURM batch script may include: + + .. code-block:: bash + + #SBATCH --nodes 3 + + python run_libe_forces.py --nworkers 3 + +When using **gen_on_manager**, set ``nworkers`` to the number of workers desired for running simulations. + +Dedicated Mode +^^^^^^^^^^^^^^ -Alternatively, in **distributed** mode, the libEnsemble (manager/worker) processes -will share nodes with submitted tasks. This enables libEnsemble, using the *mpi4py* -communicator, to be run with the workers spread across nodes so as to be co-located -with their tasks. +If the :ref:`libE_specs` option **dedicated_mode** is set to +True, the MPI executor will not launch applications on nodes where libEnsemble Python +processes (manager and workers) are running. Workers launch applications onto the +remaining nodes in the allocation. + +.. list-table:: + :widths: 60 40 + + * - .. image:: ../images/centralized_dedicated.png + :alt: centralized dedicated mode + :scale: 30 + + - In calling script: + + .. code-block:: python + :linenos: + + ensemble.libE_specs = LibeSpecs( + num_resource_sets=2, + dedicated_mode=True, + ) + + A SLURM batch script may include: + + .. code-block:: bash + + #SBATCH --nodes 3 + + python run_libe_forces.py --nworkers 3 + +Note that **gen_on_manager** is not set in the above example. + +Distributed Running +------------------- + +In the **distributed** approach, libEnsemble can be run using the **mpi4py** +communicator, with workers distributed across nodes. This is most often used +when workers run simulation code directly, via a Python interface. The user +script is invoked with an MPI runner, for example (using an `mpich`-based MPI):: + + mpirun -np 4 -ppn 1 python myscript.py + +The distributed approach, can also be used with the executor, to co-locate workers +with the applications they submit. Ensuring that workers are placed as required in this +case requires :ref:`a careful MPI rank placement `. .. image:: ../images/distributed_new_detailed.png :alt: distributed :scale: 30 :align: center -Configurations with multiple nodes per worker or multiple workers per node are both -common use cases. The distributed approach allows the libEnsemble worker to read files -produced by the application on local node storage. HPC systems that allow only one -application to be launched to a node at any one time prevent distributed configuration. +This allows the libEnsemble worker to read files produced by the application on +local node storage. Configuring the Run ------------------- @@ -44,41 +120,50 @@ the nodes within that allocation. *How does libEnsemble know where to run tasks (user applications)?* -The libEnsemble :doc:`Executor<../executor/ex_index>` can be initialized from the user calling +The libEnsemble :doc:`MPI Executor<../executor/mpi_executor>` can be initialized from the user calling script, and then used by workers to run tasks. The Executor will automatically detect the nodes available on most systems. Alternatively, the user can provide a file called **node_list** in the run directory. By default, the Executor will divide up the nodes evenly to each worker. -If the argument ``libE_specs["dedicated_mode"]=True`` is used when initializing libEnsemble, then any node -that is running a libEnsemble manager or worker will be removed from the node-list available -to the workers, ensuring libEnsemble has dedicated nodes. -To run in central mode using a 5-node allocation with 4 workers: From the head node -of the allocation:: +Mapping Tasks to Resources +-------------------------- + +The :ref:`resource manager` detects node lists from +:ref:`common batch schedulers`, +and partitions these to workers. The :doc:`MPI Executor<../executor/mpi_executor>` +accesses the resources available to the current worker when launching tasks. + +Zero-resource workers +--------------------- - mpirun -np 5 python myscript.py +Users with persistent ``gen_f`` functions may notice that the persistent workers +are still automatically assigned system resources. This can be resolved by using +the ``gen_on_manager`` option or by +:ref:`fixing the number of resource sets`. -or:: +Assigning GPUs +-------------- - python myscript.py --comms local --nworkers 4 +libEnsemble automatically detects and assigns Nvidia, AMD, and Intel GPUs without modifying the user scripts. This automatically works on many systems, but if the assignment is incorrect or needs to be modified the user can specify :ref:`platform information`. +The :doc:`forces_gpu tutorial<../tutorials/forces_gpu_tutorial>` shows an example of this. -Either of these will run libEnsemble (inc. manager and 4 workers) on the first node. The remaining -4 nodes will be divided among the workers for submitted applications. If the same run was -performed without ``libE_specs["dedicated_mode"]=True``, runs could be submitted to all 5 nodes. The number of workers -can be modified to allow either multiple workers to map to each node or multiple nodes per worker. +Varying resources +----------------- -To launch libEnsemble distributed requires a less trivial libEnsemble run script. -For example:: +libEnsemble also features :ref:`dynamic resource assignment`, whereby the +number of processes and/or the number of GPUs can be a set for each simulation by the generator. - mpirun -np 5 -ppn 1 python myscript.py +Overriding Auto-Detection +------------------------- + +libEnsemble can automatically detect system information. This includes resource information, such as +available nodes and the number of cores on the node, and information about available MPI runners. -would launch libEnsemble with 5 processes across 5 nodes. However, the manager would have its -own node, which is likely wasteful. More often, a ``machinefile`` is used to add the manager to -the first node. In the :doc:`examples` directory, you can find an example submission -script, configured to run libensemble distributed, with multiple workers per node or multiple nodes -per worker, and adding the manager onto the first node. +System detection for resources can be overridden using the :ref:`resource_info` +libE_specs option. -HPC systems that only allow one application to be launched to a node at any one time, -will not allow a distributed configuration. +When using the MPI Executor, it is possible to override the detected information using the +`custom_info` argument. See the :doc:`MPI Executor<../executor/mpi_executor>` for more. Systems with Launch/MOM Nodes ----------------------------- @@ -115,35 +200,9 @@ or *to entirely different systems*. (New) Multi-System: libEnsemble + BalsamExecutor -Submission scripts for running on launch/MOM nodes and for using Balsam, can be found in +Submission scripts for running on launch/MOM nodes and for using Balsam can be found in the :doc:`examples`. -Mapping Tasks to Resources --------------------------- - -The :ref:`resource manager` can :ref:`detect system resources`, -and partition these to workers. The :doc:`MPI Executor<../executor/mpi_executor>` -accesses the resources available to the current worker when launching tasks. - -Zero-resource workers -~~~~~~~~~~~~~~~~~~~~~ - -Users with persistent ``gen_f`` functions may notice that the persistent workers -are still automatically assigned system resources. This can be resolved by -:ref:`fixing the number of resource sets`. - -Overriding Auto-Detection -------------------------- - -libEnsemble can automatically detect system information. This includes resource information, such as -available nodes and the number of cores on the node, and information about available MPI runners. - -System detection for resources can be overridden using the :ref:`resource_info` -libE_specs option. - -When using the MPI Executor, it is possible to override the detected information using the -`custom_info` argument. See the :doc:`MPI Executor<../executor/mpi_executor>` for more. - .. _globus_compute_ref: Globus Compute - Remote User Functions @@ -215,7 +274,6 @@ libEnsemble on specific HPC systems. improv perlmutter polaris - spock_crusher summit srun example_scripts diff --git a/docs/platforms/polaris.rst b/docs/platforms/polaris.rst index 4cdabbcf0b..5fdf82aaae 100644 --- a/docs/platforms/polaris.rst +++ b/docs/platforms/polaris.rst @@ -39,7 +39,6 @@ environment (if you need ``conda install``). More details at `Python for Polaris See :doc:`here<../advanced_installation>` for more information on advanced options for installing libEnsemble, including using Spack. - Job Submission -------------- @@ -65,7 +64,7 @@ A simple example batch script for a libEnsemble use case that runs 5 workers cd $PBS_O_WORKDIR - python run_libe_forces.py --comms local --nworkers 5 + python run_libe_forces.py --nworkers 5 The script can be run with:: diff --git a/docs/platforms/spock_crusher.rst b/docs/platforms/spock_crusher.rst deleted file mode 100644 index e19c9120fa..0000000000 --- a/docs/platforms/spock_crusher.rst +++ /dev/null @@ -1,82 +0,0 @@ -============= -Spock/Crusher -============= - -Spock_ and Crusher_ are early-access testbed systems located at Oak Ridge -Leadership Computing Facility (OLCF). - -Each Spock compute node consists of one 64-core AMD EPYC "Rome" CPU and four -AMD MI100 GPUs. - -Each Crusher compute node contains a 64-core AMD EPYC and 4 AMD MI250X GPUs -(8 Graphics Compute Dies). - -These systems use the SLURM scheduler to submit jobs from login nodes to run on the -compute nodes. - -Configuring Python and Installation ------------------------------------ - -Begin by loading the ``python`` module:: - - module load cray-python - -Job Submission --------------- - -Slurm_ is used for job submission and management. libEnsemble runs on the -compute nodes using either ``multi-processing`` or ``mpi4py``. - -If running more than one worker per node, the following is recommended to prevent -resource conflicts:: - - export SLURM_EXACT=1 - export SLURM_MEM_PER_NODE=0 - -Installing libEnsemble and dependencies ---------------------------------------- - -libEnsemble can be installed via pip:: - - pip install libensemble - -Example -------- - -To run the :doc:`forces_gpu<../tutorials/forces_gpu_tutorial>` tutorial on Spock or Crusher. - -To obtain the example you can git clone libEnsemble - although only -the forces sub-directory is needed:: - - git clone https://github.com/Libensemble/libensemble - cd libensemble/libensemble/tests/scaling_tests/forces/forces_app - -To compile forces (in addition to cray-python module):: - - module load rocm - module load craype-accel-amd-gfx90a # (craype-accel-amd-gfx908 on Spock) - cc -DGPU -I${ROCM_PATH}/include -L${ROCM_PATH}/lib -lamdhip64 -fopenmp -O3 -o forces.x forces.c - -Now go to forces_gpu directory:: - - cd ../forces_gpu - -Now grab an interactive session on one node:: - - salloc --nodes=1 -A --time=00:10:00 - -Then in the session run:: - - python run_libe_forces.py -n 5 - -This places the generator on the first worker and runs simulations on the -others (each simulation using one GPU). - -To see GPU usage, ssh into the node you are on in another window and run:: - - module load rocm - watch -n 0.1 rocm-smi - -.. _Crusher: https://docs.olcf.ornl.gov/systems/crusher_quick_start_guide.html -.. _Slurm: https://slurm.schedmd.com/ -.. _Spock: https://docs.olcf.ornl.gov/systems/spock_quick_start_guide.html diff --git a/docs/platforms/summit.rst b/docs/platforms/summit.rst index b68f9c35ee..9a08a21eb5 100644 --- a/docs/platforms/summit.rst +++ b/docs/platforms/summit.rst @@ -1,17 +1,20 @@ -====== -Summit -====== +======================= +Summit (Decommissioned) +======================= -Summit_ is an IBM AC922 system located at the Oak Ridge Leadership Computing -Facility (OLCF). Each of the approximately 4,600 compute nodes on Summit contains two +Summit_ was an IBM AC922 system located at the Oak Ridge Leadership Computing +Facility (OLCF). Each of the approximately 4,600 compute nodes on Summit contained two IBM POWER9 processors and six NVIDIA Volta V100 accelerators. -Summit features three tiers of nodes: login, launch, and compute nodes. +Summit featured three tiers of nodes: login, launch, and compute nodes. Users on login nodes submit batch runs to the launch nodes. Batch scripts and interactive sessions run on the launch nodes. Only the launch nodes can submit MPI runs to the compute nodes via ``jsrun``. +These docs are maintained to guide libEnsemble's usage on three-tier systems and/or +`jsrun` systems similar to Summit. + Configuring Python ------------------ @@ -21,7 +24,7 @@ Begin by loading the Python 3 Anaconda module:: You can now create and activate your own custom conda_ environment:: - conda create --name myenv python=3.9 + conda create --name myenv python=3.10 export PYTHONNOUSERSITE=1 # Make sure get python from conda env . activate myenv @@ -57,13 +60,13 @@ Or, you can install via ``conda``: See :doc:`here<../advanced_installation>` for more information on advanced options for installing libEnsemble. - Special note on resource sets and Executor submit options + --------------------------------------------------------- When using the portable MPI run configuration options (e.g., num_nodes) to the :doc:`MPIExecutor<../executor/mpi_executor>` ``submit`` function, it is important -to note that, due to the `resource sets`_ used on Summit, the options refer to +to note that, due to the resource sets used on Summit, the options refer to resource sets as follows: - num_procs (int, optional) – The total number resource sets for this run. @@ -114,7 +117,7 @@ available on a Summit node, and thus two such tasks may be allocated to each nod Job Submission -------------- -Summit uses LSF_ for job management and submission. For libEnsemble, the most +Summit used LSF_ for job management and submission. For libEnsemble, the most important command is ``bsub`` for submitting batch scripts from the login nodes to execute on the launch nodes. @@ -191,20 +194,13 @@ Launching User Applications from libEnsemble Workers ---------------------------------------------------- Only the launch nodes can submit MPI runs to the compute nodes via ``jsrun``. -This can be accomplished in user ``sim_f`` functions directly. However, it is highly +This can be accomplished in user simulator functions directly. However, it is highly recommended that the :doc:`Executor<../executor/ex_index>` interface -be used inside the ``sim_f`` or ``gen_f``, because this provides a portable interface +be used inside the simulator or generator, because this provides a portable interface with many advantages including automatic resource detection, portability, launch failure resilience, and ease of use. -Additional Information ----------------------- - -See the OLCF guides_ for more information about Summit. - .. _conda: https://conda.io/en/latest/ -.. _guides: https://docs.olcf.ornl.gov/systems/summit_user_guide.html .. _LSF: https://www.olcf.ornl.gov/wp-content/uploads/2018/12/summit_workshop_fuson.pdf .. _mpi4py: https://mpi4py.readthedocs.io/en/stable/ -.. _resource sets: https://docs.olcf.ornl.gov/systems/summit_user_guide.html#job-launcher-jsrun -.. _Summit: https://docs.olcf.ornl.gov/systems/summit_user_guide.html +.. _Summit: https://www.olcf.ornl.gov/olcf-resources/compute-systems/summit/ diff --git a/docs/resource_manager/overview.rst b/docs/resource_manager/overview.rst index d68e35567b..57231962b9 100644 --- a/docs/resource_manager/overview.rst +++ b/docs/resource_manager/overview.rst @@ -276,7 +276,7 @@ Also, this can be set on the command line as a convenience. .. code-block:: bash - python run_ensemble.py --comms local --nworkers 5 --nresource_sets 8 + python run_ensemble.py --nworkers 5 --nresource_sets 8 .. _persistent_sampling_var_resources.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/gen_funcs/persistent_sampling_var_resources.py .. _test_GPU_variable_resources.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_GPU_variable_resources.py diff --git a/docs/resource_manager/resource_detection.rst b/docs/resource_manager/resource_detection.rst index 461d1d0eed..2048eb2793 100644 --- a/docs/resource_manager/resource_detection.rst +++ b/docs/resource_manager/resource_detection.rst @@ -18,21 +18,21 @@ LSF LSB_HOSTS/LSB_MCPU_HOSTS PBS PBS_NODEFILE =========== =========================== -These environment variable names can be modified via the :ref:`resource_info` +These environment variable names can be modified via the :ref:`resource_info` :class:`libE_specs` option. -On other systems you may have to supply a node list in a file called **node_list** -in your run directory. For example, on ALCF system Cooley_, the session node list +On other systems, you may have to supply a node list in a file called **node_list** +in your run directory. For example, on the ALCF system Cooley_, the session node list can be obtained as follows:: cat $COBALT_NODEFILE > node_list Resource detection can be disabled by setting -``libE_specs["disable_resource_manager"] = True``, and users can simply supply run +``libE_specs["disable_resource_manager"] = True``, and users can supply run configuration options on the Executor submit line. This will usually work sufficiently on -systems that have application-level scheduling and queuing (e.g., ``jsrun`` on Summit). +systems that have application-level scheduling and queuing (e.g., ``jsrun``). However, on many cluster and multi-node systems, if the built-in resource manager is disabled, then runs without a hostlist or machinefile supplied may be undesirably scheduled to the same nodes. diff --git a/docs/resource_manager/zero_resource_workers.rst b/docs/resource_manager/zero_resource_workers.rst index fe8c243eb1..4c72cf5d7b 100644 --- a/docs/resource_manager/zero_resource_workers.rst +++ b/docs/resource_manager/zero_resource_workers.rst @@ -9,7 +9,7 @@ only run ``gen_f`` functions in-place (i.e., they do not use the Executor to submit applications to allocated nodes). Suppose the user is using the :meth:`parse_args()` function and runs:: - python run_ensemble_persistent_gen.py --comms local --nworkers 3 + python run_ensemble_persistent_gen.py --nworkers 3 If three nodes are available in the node allocation, the result may look like the following. @@ -21,7 +21,7 @@ following. To avoid the the wasted node above, add an extra worker:: - python run_ensemble_persistent_gen.py --comms local --nworkers 4 + python run_ensemble_persistent_gen.py --nworkers 4 and in the calling script (*run_ensemble_persistent_gen.py*), explicitly set the number of resource sets to the number of workers that will be running simulations. @@ -53,7 +53,7 @@ concurrency desired by the ensemble, taking into account generators and simulato Users can set generator resources using the *libE_specs* options ``gen_num_procs`` and/or ``gen_num_gpus``, which take integer values. -If only ``gen_num_gpus`` is set, then the number of processors is set to match. +If only ``gen_num_gpus`` is set, then the number of processors is set to match. To vary generator resources, ``persis_info`` settings can be used in allocation functions before calling the ``gen_work`` support function. This takes the diff --git a/docs/running_libE.rst b/docs/running_libE.rst index cb86262526..ae658e31c6 100644 --- a/docs/running_libE.rst +++ b/docs/running_libE.rst @@ -53,7 +53,7 @@ supercomputers. or an :class:`Ensemble` object with ``Ensemble(parse_args=True)``, you can specify these on the command line:: - python myscript.py --comms local --nworkers N + python myscript.py --nworkers N This will launch one manager and ``N`` workers. diff --git a/docs/tutorials/aposmm_tutorial.rst b/docs/tutorials/aposmm_tutorial.rst index 2caf4b3cd9..84f549753a 100644 --- a/docs/tutorials/aposmm_tutorial.rst +++ b/docs/tutorials/aposmm_tutorial.rst @@ -117,7 +117,7 @@ busy. In practice, since a single worker becomes "persistent" for APOSMM, users should initiate one more worker than the number of parallel simulations:: - python my_aposmm_routine.py --comms local --nworkers 4 + python my_aposmm_routine.py --nworkers 4 results in three workers running simulations and one running APSOMM. @@ -265,7 +265,7 @@ optimization method:: Finally, run this libEnsemble / APOSMM optimization routine with the following:: - python my_first_aposmm.py --comms local --nworkers 4 + python my_first_aposmm.py --nworkers 4 Please note that one worker will be "persistent" for APOSMM for the duration of the routine. diff --git a/docs/tutorials/executor_forces_tutorial.rst b/docs/tutorials/executor_forces_tutorial.rst index 914edaf2a2..e01496734b 100644 --- a/docs/tutorials/executor_forces_tutorial.rst +++ b/docs/tutorials/executor_forces_tutorial.rst @@ -32,7 +32,7 @@ Assuming MPI and its C compiler ``mpicc`` are available, compile mpicc -O3 -o forces.x forces.c -lm -Alternative build lines for different platforms can be found in the ``build_forces.sh`` +Alternative build lines for different platforms can be found in the build_forces.sh_ file in the same directory. Calling Script @@ -211,7 +211,7 @@ This completes our calling script and simulation function. Run libEnsemble with: .. code-block:: bash - python run_libe_forces.py --comms local --nworkers [nworkers] + python run_libe_forces.py --nworkers [nworkers] where ``nworkers`` is one more than the number of concurrent simulations. @@ -226,7 +226,7 @@ and ``ensemble.log`` as usual. .. code-block:: bash - python run_libe_forces.py --comms local --nworkers 3 + python run_libe_forces.py --nworkers 3 my ``libE_stats.txt`` resembled:: @@ -362,13 +362,13 @@ E.g., Instead of: .. code-block:: bash - python run_libe_forces.py --comms local --nworkers 5 + python run_libe_forces.py --nworkers 5 use: .. code-block:: bash - python run_libe_forces.py --comms local --nworkers 4 + python run_libe_forces.py --nworkers 4 Note that as the generator random number seed will be zero instead of one, the checksum will change. @@ -380,7 +380,7 @@ Running forces application with input file Many applications read an input file instead of being given parameters directly on the run line. forces_simple_with_input_file_ directory contains a variant of this example, where a templated -input file is parametized for each evaluation. +input file is parameterized for each evaluation. This requires **jinja2** to be installed:: @@ -514,6 +514,7 @@ these parameters can also be specified as different variables with corresponding to generator and simulator functions. .. _examples/tutorials/forces_with_executor: https://github.com/Libensemble/libensemble/tree/develop/examples/tutorials/forces_with_executor +.. _build_forces.sh: https://github.com/Libensemble/libensemble/blob/main/libensemble/tests/scaling_tests/forces/forces_app/build_forces.sh .. _forces_app: https://github.com/Libensemble/libensemble/tree/main/libensemble/tests/scaling_tests/forces/forces_app .. _forces_simple: https://github.com/Libensemble/libensemble/tree/main/libensemble/tests/scaling_tests/forces/forces_simple .. _forces_simple_with_input_file: https://github.com/Libensemble/libensemble/tree/main/libensemble/tests/scaling_tests/forces/forces_simple_with_input_file diff --git a/docs/tutorials/forces_gpu_tutorial.rst b/docs/tutorials/forces_gpu_tutorial.rst index ba425e8363..be487f33cc 100644 --- a/docs/tutorials/forces_gpu_tutorial.rst +++ b/docs/tutorials/forces_gpu_tutorial.rst @@ -7,9 +7,9 @@ to the GPU. The libEnsemble scripts in this example are available under forces_gpu_ in the libEnsemble repository. This example is based on the -:doc:`simple forces tutorial <../tutorials/executor_forces_tutorial>` with +:doc:`simple forces tutorial <../tutorials/executor_forces_tutorial>` with a slightly modified simulation function (to assign GPUs) and a greatly increased -number of particles (allows live GPU usage to be viewed). +number of particles (to allow real-time GPU usage to be viewed). In the first example, each worker will be using one GPU. The code will assign the GPUs available to each worker, using the appropriate method. This works on systems @@ -35,7 +35,6 @@ from the simple forces example are highlighted: # Optional - to print GPU settings from libensemble.tools.test_support import check_gpu_setting - def run_forces(H, persis_info, sim_specs, libE_info): """Launches the forces MPI app and auto-assigns ranks and GPU resources. @@ -154,7 +153,6 @@ and use this information however you want. output = np.zeros(1, dtype=sim_specs["out"]) output["energy"][0] = final_energy - return output The above code will assign a GPU to each worker on CUDA-capable systems, @@ -214,7 +212,7 @@ nine workers (the extra worker runs the persistent generator). For example:: - python run_libe_forces.py --comms local --nworkers 9 + python run_libe_forces.py --nworkers 9 See :ref:`zero-resource workers` for more ways to express this. @@ -225,6 +223,8 @@ If you want to have two GPUs per worker on the same system (with four GPUs per n you could assign only four workers. You will see that two GPUs are used for each forces run. +.. _var-resources-gpu: + Varying resources ----------------- @@ -298,7 +298,7 @@ that runs 8 workers on 2 nodes: export MPICH_GPU_SUPPORT_ENABLED=1 export SLURM_EXACT=1 - python run_libe_forces.py --comms local --nworkers 9 + python run_libe_forces.py --nworkers 9 where ``SLURM_EXACT`` is set to help prevent resource conflicts on each node. diff --git a/docs/tutorials/gpcam_tutorial.rst b/docs/tutorials/gpcam_tutorial.rst new file mode 100644 index 0000000000..a013c1b67e --- /dev/null +++ b/docs/tutorials/gpcam_tutorial.rst @@ -0,0 +1,340 @@ +Surrogate Modeling with gpCAM +============================= + +This example uses gpCAM_ to construct a global surrogate of ``f`` values using a Gaussian process. + +In each iteration, a batch of points is produced for concurrent evaluation, maximizing uncertainty reduction. + +|Open in Colab| + +Ensure that libEnsemble, and gpCAM are installed via: ``pip install libensemble gpcam`` + +Generator function +------------------ + +The gpCAM generator function is called ``persistent_gpCAM``. + +This persistent generator is started at the beginning of the Ensemble and runs until the Ensemble closes down. + +This version (and others) of the gpCAM generator can be found at `libensemble/gen_funcs/persistent_gpCAM.py `_ and can be imported from that location when libEnsemble is installed as follows: + +``from libensemble.gen_funcs.persistent_gpCAM import persistent_gpCAM`` + +.. code-block:: python + + import numpy as np + from numpy.lib.recfunctions import repack_fields + from gpcam import GPOptimizer as GP + + # Standard options for manager/generator comms + from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG + from libensemble.tools.persistent_support import PersistentSupport + + def persistent_gpCAM(H_in, persis_info, gen_specs, libE_info): + """Run a batched gpCAM model to create a surrogate""" + + # Initialize + rng, batch_size, n, lb, ub, x_new, y_new, ps = _initialize_gpcAM(gen_specs["user"], libE_info) + ask_max_iter = gen_specs["user"].get("ask_max_iter") or 10 + test_points = _read_testpoints(gen_specs["user"]) + noise = 1e-8 # Initializes noise + my_gp = None + + # Start with a batch of random points + x_new = rng.uniform(lb, ub, (batch_size, n)) + H_o = np.zeros(batch_size, dtype=gen_specs["out"]) + H_o["x"] = x_new + tag, Work, calc_in = ps.send_recv(H_o) # Send random points for evaluation and wait for results + + while tag not in [STOP_TAG, PERSIS_STOP]: + y_new = np.atleast_2d(calc_in["f"]).T + my_gp = _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise) + + # Request new points + x_new = my_gp.ask( + input_set=np.column_stack((lb, ub)), + n=batch_size, + pop_size=batch_size, + acquisition_function="total correlation", + max_iter=ask_max_iter, # Larger takes longer. gpCAM default is 20. + )["x"] + + H_o = np.zeros(batch_size, dtype=gen_specs["out"]) + H_o["x"] = x_new + tag, Work, calc_in = ps.send_recv(H_o) # Send points for evaluation and wait for results + + # If final points were returned update the model + if calc_in is not None: + y_new = np.atleast_2d(calc_in["f"]).T + my_gp = _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise) + + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG + +Common acquisition functions include: + +**Uncertainty reduction:** + +- **"variance"** (default): The optimizer will produce N best points. +- **"total correlation"**: More expensive but points produced are self-avoiding. + +**Bayesian optimization:** + +These produce one point at a time unless using the `HGDL `_ option. + +- **"ucb" / "lcb"**: Upper/Lower Confidence Bound. +- **"expected improvement"**: Expected Improvement. + +For more options see: https://gpcam.lbl.gov/examples/acquisition-functions + +---- + +The following code adds the functions used by ``persistent_gpCAM``. + +``_update_gp`` is where the GP is fed the data and trained. + +.. code-block:: python + + def _initialize_gpcAM(user_specs, libE_info): + """Extract user params""" + rng_seed = user_specs.get("rng_seed") # will default to None + rng = np.random.default_rng(rng_seed) # Create random stream + b = user_specs["batch_size"] + lb = np.array(user_specs["lb"]) + ub = np.array(user_specs["ub"]) + n = len(lb) # no. of dimensions + init_x = np.empty((0, n)) + init_y = np.empty((0, 1)) + ps = PersistentSupport(libE_info, EVAL_GEN_TAG) # init comms + return rng, b, n, lb, ub, init_x, init_y, ps + + + def _read_testpoints(U): + """Read numpy file containing evaluated points for measuring GP error""" + test_points_file = U.get("test_points_file") + if test_points_file is None: + return None + test_points = np.load(test_points_file) + test_points = repack_fields(test_points[["x", "f"]]) + return test_points + + + def _compare_testpoints(my_gp, test_points, persis_info): + """Compare model at test points""" + if test_points is None: + return + f_est = my_gp.posterior_mean(test_points["x"])["f(x)"] + mse = np.mean((f_est - test_points["f"]) ** 2) + persis_info.setdefault("mean_squared_error", []).append(float(mse)) + + + def _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise): + """Update Gaussian process with new points and train""" + noise_arr = noise * np.ones(len(y_new)) # Initializes noise + if my_gp is None: + my_gp = GP(x_new, y_new.flatten(), noise_variances=noise_arr) + else: + my_gp.tell(x_new, y_new.flatten(), noise_variances=noise_arr, append=True) + my_gp.train() + + if test_points is not None: + _compare_testpoints(my_gp, test_points, persis_info) + + return my_gp + +Simulator function +------------------ + +Simulator functions or ``sim_f``\ s perform calculations based on parameters created in the generator function. +Each worker runs a copy of this function in parallel. + +The function here is the simple 2D ``six_hump_camel``, for demonstration purposes. + +For running applications using parallel resources in the simulator see the `forces examples `_. + +.. code-block:: python + + # Define our simulation function + import numpy as np + + def six_hump_camel(H, persis_info, sim_specs, _): + """Six-Hump Camel sim_f.""" + + batch = len(H["x"]) # Num evaluations each sim_f call. + H_o = np.zeros(batch, dtype=sim_specs["out"]) # Define output array H + + for i, x in enumerate(H["x"]): + H_o["f"][i] = six_hump_camel_func(x) # Function evaluations placed into H + + return H_o, persis_info + + + def six_hump_camel_func(x): + """Six-Hump Camel function definition""" + x1 = x[0] + x2 = x[1] + term1 = (4 - 2.1 * x1**2 + (x1**4) / 3) * x1**2 + term2 = x1 * x2 + term3 = (-4 + 4 * x2**2) * x2**2 + + return term1 + term2 + term3 + +Calling Script +-------------- + +Our calling script configures libEnsemble, the generator function, and the simulator function. It then create the ensemble object and runs the ensemble. + +First we will create a cleanup script so we can easily re-run. + +.. code-block:: python + + # To rerun this notebook, we need to delete the ensemble directory. + import shutil + def cleanup(): + try: + shutil.rmtree("ensemble") + except: + pass + +This calling script imports the Gen and Sim functions from the locations in the installed libensemble package. +If you wish to make your own functions based on the above, those can be imported instead. + +.. code-block:: python + + import numpy as np + from pprint import pprint + + from libensemble import Ensemble + from libensemble.specs import LibeSpecs, GenSpecs, SimSpecs, AllocSpecs, ExitCriteria + + # If importing from libensemble + from libensemble.gen_funcs.persistent_gpCAM import persistent_gpCAM + from libensemble.sim_funcs.six_hump_camel import six_hump_camel + + from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens + import warnings + + warnings.filterwarnings("ignore", message="Default hyperparameter_bounds") + warnings.filterwarnings("ignore", message="Hyperparameters initialized") + + nworkers = 4 + + # When using gen_on_manager, nworkers is number of concurrent sims. + # final_gen_send means the last evaluated points are returned to the generator to update the model. + libE_specs = LibeSpecs(nworkers=nworkers, gen_on_manager=True, final_gen_send=True) + + n = 2 # Input dimensions + batch_size = 4 + num_batches = 6 + + gen_specs = GenSpecs( + gen_f=persistent_gpCAM, # Generator function + persis_in=["f"], # Objective, defined in sim, is returned to gen + outputs=[("x", float, (n,))], # Parameters (name, type, size) + user={ + "batch_size": batch_size, + "lb": np.array([-2, -1]), # lower boundaries for n dimensions + "ub": np.array([2, 1]), # upper boundaries for n dimensions + "ask_max_iter": 5, # Number of iterations for ask (default 20) + "rng_seed": 0, + }, + ) + + sim_specs = SimSpecs( + sim_f=six_hump_camel, # Simulator function + inputs=["x"], # Input field names. "x" defined in gen + outputs=[("f", float)], # Objective + ) + + # Starts one persistent generator. Simulated values are returned in batch. + alloc_specs = AllocSpecs( + alloc_f=only_persistent_gens, + user={"async_return": False}, # False = batch returns + ) + + exit_criteria = ExitCriteria(sim_max=num_batches*batch_size) + + # Initialize and run the ensemble. + ensemble = Ensemble( + libE_specs=libE_specs, + sim_specs=sim_specs, + gen_specs=gen_specs, + alloc_specs=alloc_specs, + exit_criteria=exit_criteria, + ) + +At the end of our calling script we run the ensemble. + +.. code-block:: python + + # To ensure re-running works - clean output and reset any persistent information + cleanup() + ensemble.persis_info = {} + + H, persis_info, flag = ensemble.run() # Start the ensemble. Blocks until completion. + ensemble.save_output("H_array", append_attrs=False) # Save H (history of all evaluated points) to file + pprint(H[["sim_id", "x", "f"]][:16]) # See first 16 results + +Rerun and test model at known points +------------------------------------ + +To see how the accuracy of the surrogate model improves, we can use previously evaluated points as test points and run again with a different seed. + +.. code-block:: python + + ensemble.gen_specs.user["rng_seed"] = 123 + ensemble.gen_specs.user["test_points_file"] = "H_array.npy" # our previous file + + # To ensure re-running works - clean output and reset any persistent information + cleanup() + ensemble.persis_info = {} + + H, persis_info, flag = ensemble.run() + print(persis_info) + +Viewing model progression +------------------------- + +Now we can check how our model's values compared against the values at known test points as the ensemble progresses. +The comparison is based on the **mean squared error** between the gpCAM model and our known +values at the test points. + +.. code-block:: python + + import matplotlib + import matplotlib.pyplot as plt + + # Get "mean_squared_error" from generators return (worker 0 as we ran gen_on_manager) + mse = persis_info[0]["mean_squared_error"] + niter = len(mse) + num_sims = list(range(batch_size, (niter * batch_size) + 1, batch_size)) + + # Plotting the data + markersize = 10 + plt.figure(figsize=(10, 5)) + plt.plot( + num_sims, mse, marker="^", markeredgecolor="black", markeredgewidth=2, + markersize=markersize, linewidth=2, label="Mean squared error" + ) + plt.xticks(num_sims) + + # Labeling the axes and the legend + plt.title('Mean Squared Error at test points') + plt.xlabel("Number of simulations") + plt.ylabel('Mean squared error (rad$^2$)') + legend = plt.legend(framealpha=1, edgecolor="black") # Increase edge width here + plt.grid(True) + plt.show() + +The plot should look similar to the following. + +.. note:: + The graph may differ between runs because, although we seed libEnsemble's random number generator, + gpCAM introduces some randomness when initializing hyperparameters. + +.. image:: ../images/gpcam_model_improvement.png + :alt: gpcam_model_improvement + :align: center + +.. _gpCAM: https://github.com/lbl-camera/gpCAM +.. |Open in Colab| image:: https://colab.research.google.com/assets/colab-badge.svg + :target: http://colab.research.google.com/github/Libensemble/libensemble/blob/develop/examples/tutorials/gpcam_surrogate_model/gpcam.ipynb diff --git a/docs/tutorials/local_sine_tutorial.rst b/docs/tutorials/local_sine_tutorial.rst index 00dafdcdfb..0509824750 100644 --- a/docs/tutorials/local_sine_tutorial.rst +++ b/docs/tutorials/local_sine_tutorial.rst @@ -22,7 +22,7 @@ need to write a new allocation function. .. code-block:: bash - python --version # This should be >= 3.9 + python --version # This should be >= 3.10 .. _Python: https://www.python.org/ diff --git a/docs/tutorials/tutorials.rst b/docs/tutorials/tutorials.rst index f114a7afa0..6b40e4b658 100644 --- a/docs/tutorials/tutorials.rst +++ b/docs/tutorials/tutorials.rst @@ -6,5 +6,6 @@ Tutorials local_sine_tutorial executor_forces_tutorial forces_gpu_tutorial + gpcam_tutorial aposmm_tutorial calib_cancel_tutorial diff --git a/examples/README.rst b/examples/README.rst index eced66ddce..3c97602a20 100644 --- a/examples/README.rst +++ b/examples/README.rst @@ -12,9 +12,9 @@ If you wish to clone libEnsemble to try the examples instead of installing from pip install -e . cd libensemble/tests/regression_tests -Any of the tests can be run similarly to the following:: +Any of the tests can be run similarly to the following (-n is also short for --nworkers):: - python test_uniform_sampling.py --comms local --nworkers 3 + python test_uniform_sampling.py --nworkers 3 The command line arguments are parsed by a ``parse_args`` module within each of the scripts. If you have ``mpi4py`` installed you can alternatively run with:: diff --git a/examples/libE_submission_scripts/bebop_submit_pbs_central.sh b/examples/libE_submission_scripts/bebop_submit_pbs_central.sh new file mode 100644 index 0000000000..7fb474194f --- /dev/null +++ b/examples/libE_submission_scripts/bebop_submit_pbs_central.sh @@ -0,0 +1,13 @@ +#!/bin/bash -l +#PBS -l select=4 +#PBS -l walltime=00:15:00 +#PBS -q bdwall +#PBS -A [project] +#PBS -N libE_example + +cd $PBS_O_WORKDIR +# Choose MPI backend. Note that the built mpi4py in your environment should match. +module load oneapi/mpi +# module load openmpi + +python run_libe_example.py -n 16 diff --git a/examples/libE_submission_scripts/bebop_submit_slurm_central.sh b/examples/libE_submission_scripts/bebop_submit_slurm_central.sh deleted file mode 100644 index 011db26eb0..0000000000 --- a/examples/libE_submission_scripts/bebop_submit_slurm_central.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -#SBATCH -J libE_test_central -#SBATCH -N 5 -#SBATCH -p knlall -#SBATCH -A -#SBATCH -o tlib.%j.%N.out -#SBATCH -e tlib.%j.%N.error -#SBATCH -t 01:00:00 - -# Launch script for running in central mode with mpi4py. -# libEnsemble will run on a dedicated node (or nodes). -# The remaining nodes in the allocation will be dedicated to worker launched apps. -# Use executor with auto-resources=True and central_mode=True. - -# User to edit these variables -export EXE=libE_calling_script.py -export NUM_WORKERS=4 -export I_MPI_FABRICS=shm:tmi - -# Overcommit will allow ntasks up to the no. of contexts on one node (eg. 320 on Bebop) -srun --overcommit --ntasks=$(($NUM_WORKERS+1)) --nodes=1 python $EXE - -# To use local mode instead of mpi4py (with parse_args()) -# python calling_script.py --comms local --nworkers $NUM_WORKERS diff --git a/examples/libE_submission_scripts/bridges_submit_slurm_central.sh b/examples/libE_submission_scripts/bridges_submit_slurm_central.sh index d00f3d0417..85fff856e3 100644 --- a/examples/libE_submission_scripts/bridges_submit_slurm_central.sh +++ b/examples/libE_submission_scripts/bridges_submit_slurm_central.sh @@ -19,4 +19,4 @@ export NUM_WORKERS=4 mpirun -np $(($NUM_WORKERS+1)) -ppn $(($NUM_WORKERS+1)) python $EXE # To use local mode instead of mpi4py (with parse_args()) -# python $EXE --comms local --nworkers $NUM_WORKERS +# python $EXE -n $NUM_WORKERS diff --git a/examples/libE_submission_scripts/cobalt_submit_mproc.sh b/examples/libE_submission_scripts/cobalt_submit_mproc.sh deleted file mode 100755 index d2fa3a10a4..0000000000 --- a/examples/libE_submission_scripts/cobalt_submit_mproc.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -x -#COBALT -t 00:30:00 -#COBALT -O libE_mproc_MOM -#COBALT -n 4 -#COBALT -q debug-flat-quad # Up to 8 nodes only -##COBALT -q default # For large jobs >=128 nodes -##COBALT -A - -# Script to run libEnsemble using multiprocessing on launch nodes. -# Assumes Conda environment is set up. - -# To be run with central job management -# - Manager and workers run on launch node. -# - Workers submit tasks to the compute nodes in the allocation. - -# Name of calling script -export EXE=libE_calling_script.py - -# Communication Method -export COMMS="--comms local" - -# Number of workers. -export NWORKERS="--nworkers 4" - -# Wallclock for libE (allow clean shutdown) -export LIBE_WALLCLOCK=25 # Optional if pass to script - -# Name of Conda environment -export CONDA_ENV_NAME= - -# Conda location - theta specific -export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH -export LD_LIBRARY_PATH=~/.conda/envs/$CONDA_ENV_NAME/lib:$LD_LIBRARY_PATH -export PMI_NO_FORK=1 # Required for python kills on Theta - -# Unload Theta modules that may interfere with job monitoring/kills -module unload trackdeps -module unload darshan -module unload xalt - -# Activate conda environment -export PYTHONNOUSERSITE=1 -. activate $CONDA_ENV_NAME - -# Launch libE -# python $EXE $NUM_WORKERS > out.txt 2>&1 # No args. All defined in calling script -# python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If calling script is using parse_args() -python $EXE $LIBE_WALLCLOCK $COMMS $NWORKERS > out.txt 2>&1 # If calling script takes wall-clock as positional arg. diff --git a/examples/libE_submission_scripts/edtb_submit_pbspro_central.sh b/examples/libE_submission_scripts/edtb_submit_pbspro_central.sh deleted file mode 100644 index 2229820a99..0000000000 --- a/examples/libE_submission_scripts/edtb_submit_pbspro_central.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -x -#PBS -l walltime=00:30:00 -#PBS -l select=4:ncpus=128 -#PBS -A - -# Script to run libEnsemble using multiprocessing on launch nodes. -# Assumes Conda environment is set up. - -# To be run with central job management -# - Manager and workers run on launch node. -# - Workers submit tasks to the compute nodes in the allocation. - -# Name of calling script - -source ~/.bashrc - -export EXE= - -# Communication Method -export COMMS="--comms local" - -# Number of workers. -export NWORKERS="--nworkers 4" - -# Name of Conda environment -export CONDA_ENV_NAME= - -# Activate conda environment -export PYTHONNOUSERSITE=1 -conda activate $CONDA_ENV_NAME - -cd $PBS_O_WORKDIR - -# openmpi on edtb -export LD_LIBRARY_PATH=/lus/theta-fs0/software/edtb/openmpi/4.1.1/lib:$LD_LIBRARY_PATH -export PATH=/lus/theta-fs0/software/edtb/openmpi/4.1.1/bin:$PATH - -# Launch libE -python $EXE $COMMS $NWORKERS > out.txt 2>&1 diff --git a/examples/libE_submission_scripts/bebop_submit_slurm_distrib.sh b/examples/libE_submission_scripts/submit_distrib_mpi4py.sh similarity index 95% rename from examples/libE_submission_scripts/bebop_submit_slurm_distrib.sh rename to examples/libE_submission_scripts/submit_distrib_mpi4py.sh index f30e84340a..2d15c07abb 100644 --- a/examples/libE_submission_scripts/bebop_submit_slurm_distrib.sh +++ b/examples/libE_submission_scripts/submit_distrib_mpi4py.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -J libE_test #SBATCH -N 4 -#SBATCH -p knlall +#SBATCH -p [queue] #SBATCH -A #SBATCH -o tlib.%j.%N.out #SBATCH -e tlib.%j.%N.error @@ -20,10 +20,11 @@ export NUM_WORKERS=4 export MANAGER_NODE=false # true = Manager has a dedicated node (assign one extra) export USE_NODE_LIST=true # If false, allow libE to determine node_list from environment. +# Sometimes may be necessary # As libE shares nodes with user applications allow fallback if contexts overrun. -unset I_MPI_FABRICS -export I_MPI_FABRICS_LIST=tmi,tcp -export I_MPI_FALLBACK=1 +# unset I_MPI_FABRICS +# export I_MPI_FABRICS_LIST=tmi,tcp +# export I_MPI_FALLBACK=1 # If using in calling script (After N mins manager kills workers and exits cleanly) export LIBE_WALLCLOCK=55 diff --git a/examples/libE_submission_scripts/submit_frontier_large.sh b/examples/libE_submission_scripts/submit_frontier_large.sh new file mode 100644 index 0000000000..0f755179f7 --- /dev/null +++ b/examples/libE_submission_scripts/submit_frontier_large.sh @@ -0,0 +1,11 @@ +#!/bin/bash +#SBATCH -J libE_warpX_full_sim_32x40 +#SBATCH -A +#SBATCH -p batch +#SBATCH --time 6:00:00 +#SBATCH --nodes 240 + +module load cray-python + +# Run one gen and 40 sim workers (6 nodes = 48 GPUs each) +python run_gpcam_warpx.py -n 41 diff --git a/examples/libE_submission_scripts/submit_lsf_simple.sh b/examples/libE_submission_scripts/submit_lsf_simple.sh new file mode 100644 index 0000000000..bfd86331e9 --- /dev/null +++ b/examples/libE_submission_scripts/submit_lsf_simple.sh @@ -0,0 +1,7 @@ +#!/bin/bash -l +#BSUB -P +#BSUB -J libe_mproc +#BSUB -W 15 +#BSUB -nnodes 2 + +python run_libe_forces.py -n 8 diff --git a/examples/libE_submission_scripts/submit_pbs_aurora.sh b/examples/libE_submission_scripts/submit_pbs_aurora.sh index b6b3625d06..0dfa59f6b6 100644 --- a/examples/libE_submission_scripts/submit_pbs_aurora.sh +++ b/examples/libE_submission_scripts/submit_pbs_aurora.sh @@ -4,11 +4,14 @@ #PBS -q #PBS -A -module use /soft/modulefiles module load frameworks export MPICH_GPU_SUPPORT_ENABLED=1 cd $PBS_O_WORKDIR # 2 nodes - 12 sim workers (6 GPUs per node) -python libE_calling_script.py --comms local --nworkers 13 +python libE_calling_script.py -n 13 + +# if using libE_specs["use_tiles_as_gpus"] = True +# 2 nodes 24 sim workers (12 GPU tiles per node) libE_specs["use_tiles_as_gpus"] = True +# python libE_calling_script.py -n 25 diff --git a/examples/libE_submission_scripts/submit_pbs_polaris.sh b/examples/libE_submission_scripts/submit_pbs_polaris.sh index 0746450ec6..f89d59824a 100644 --- a/examples/libE_submission_scripts/submit_pbs_polaris.sh +++ b/examples/libE_submission_scripts/submit_pbs_polaris.sh @@ -7,4 +7,4 @@ export MPICH_GPU_SUPPORT_ENABLED=1 cd $PBS_O_WORKDIR -python libE_calling_script.py --comms local --nworkers 4 +python libE_calling_script.py -n 4 diff --git a/examples/libE_submission_scripts/submit_pbs_simple.sh b/examples/libE_submission_scripts/submit_pbs_simple.sh index 161d38d40a..9e2f4e901b 100644 --- a/examples/libE_submission_scripts/submit_pbs_simple.sh +++ b/examples/libE_submission_scripts/submit_pbs_simple.sh @@ -7,4 +7,4 @@ # We selected 2 nodes - now running with 8 workers. export MPICH_GPU_SUPPORT_ENABLED=1 cd $PBS_O_WORKDIR -python run_libe_forces.py --comms local --nworkers 8 +python libE_calling_script.py -n 8 diff --git a/examples/libE_submission_scripts/submit_perlmutter.sh b/examples/libE_submission_scripts/submit_perlmutter.sh new file mode 100644 index 0000000000..8a029761ab --- /dev/null +++ b/examples/libE_submission_scripts/submit_perlmutter.sh @@ -0,0 +1,15 @@ +#!/bin/bash +#SBATCH -J libE_small_test +#SBATCH -A +#SBATCH -C gpu +#SBATCH --time 10 +#SBATCH --nodes 1 + +# This script is using GPU partition +export MPICH_GPU_SUPPORT_ENABLED=1 + +# One worker for generator and 4 for sims (one GPU each) +python libe_calling_script.py -n 5 + +# Or if libE_specs option gen_on_manager=True +# python libe_calling_script.py -n 4 diff --git a/examples/libE_submission_scripts/submit_slurm_simple.sh b/examples/libE_submission_scripts/submit_slurm_simple.sh index e508acc9b6..79171f1e2a 100644 --- a/examples/libE_submission_scripts/submit_slurm_simple.sh +++ b/examples/libE_submission_scripts/submit_slurm_simple.sh @@ -12,4 +12,4 @@ export SLURM_EXACT=1 export SLURM_MEM_PER_NODE=0 -python libe_calling_script.py --comms local --nworkers 8 +python libe_calling_script.py -n 8 diff --git a/examples/tutorials/aposmm/aposmm_tutorial_notebook.ipynb b/examples/tutorials/aposmm/aposmm_tutorial_notebook.ipynb index 7898ba7661..594057f3c0 100644 --- a/examples/tutorials/aposmm/aposmm_tutorial_notebook.ipynb +++ b/examples/tutorials/aposmm/aposmm_tutorial_notebook.ipynb @@ -6,7 +6,7 @@ "source": [ "# Parallel Optimization with APOSMM\n", "\n", - "This tutorial demonstrates libEnsemble’s capability to identify multiple minima of simulation output using the built-in APOSMM (Asynchronously Parallel Optimization Solver for finding Multiple Minima) generator function (`gen_f`). In this tutorial, we’ll create a simple simulation function (`sim_f`) that defines a function with multiple minima, then write a libEnsemble calling script that imports APOSMM and parameterizes it to check for minima over a domain of outputs from our `sim_f`.\n", + "This tutorial demonstrates libEnsemble’s capability to identify multiple minima from simulation outputs using the built-in APOSMM (Asynchronously Parallel Optimization Solver for finding Multiple Minima) generator function (`gen_f`). In this tutorial, we’ll create a simple simulation function (`sim_f`) that defines a function with multiple minima, then write a libEnsemble calling script that imports APOSMM and parameterizes it to check for minima over a domain of outputs from our `sim_f`.\n", "\n", "Besides libEnsemble and NumPy, SciPy and mpmath are also required dependencies.\n", "\n", @@ -43,7 +43,6 @@ "outputs": [], "source": [ "# Define our simulation function\n", - "\n", "import numpy as np\n", "\n", "\n", @@ -122,9 +121,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This allocation function starts a single Persistent APOSMM routine and provides ``sim_f`` output for points requested by APOSMM. Points can be sampled points or points from local optimization runs.\n", + "This allocation function starts a single Persistent APOSMM generator to generate points (simulation input parameters), and returns the resulting values from each simulation (run in parallel). Points can be sampled points or points from the parallel local optimization runs.\n", "\n", - "APOSMM supports a wide variety of external optimizers. The following statements set optimizer settings to ``'scipy'`` to indicate to APOSMM which optimization method to use, and help prevent unnecessary imports or package installations:" + "APOSMM supports a wide variety of external optimizers. The following statement sets the optimizer settings to ``'scipy'`` to indicate to APOSMM which optimization method to use, so it is imported at global scope:" ] }, { @@ -142,6 +141,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "This script uses the dictionary interface to configure the run (the newer object interface is equally valid).\n", "Set up ``nworkers``, ``libE_specs``, ``sim_specs``, ``gen_specs``, and ``alloc_specs``:" ] }, @@ -226,28 +226,7 @@ "source": [ "## Run the Ensemble\n", "\n", - "Optionally run the next cell to set up a live graphic of the optimization progress during execution.\n", - "\n", - "**WARNING**: The graphic may flicker when the ensemble is running." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Configure to view live progress\n", - "from libensemble.tools.live_data.plot2n import Plot2N\n", - "\n", - "libE_specs[\"live_data\"] = Plot2N(plot_type=\"2d\") # Alt: '3d'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, set `persis_info` (to provide random seeds to workers) and run the ensemble:" + "Finally, set persis_info (to provide random seeds to workers) and run the ensemble." ] }, { @@ -290,29 +269,110 @@ " \n", "The first six values correspond to the local minima for the Six-Hump Camel simulation function.\n", "\n", - "The 7th value is a repeat minimum, as APOSMM will continue to start local optimization runs.\n", + "The 7th value is a repeat minimum, as APOSMM will continue to start local optimization runs." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Viewing Animation\n", "\n", - "Please see the [API reference](https://libensemble.readthedocs.io/en/main/examples/aposmm.html) for more APOSMM configuration options and other information.\n", + "The following cell produces a 3D animation showing the random sampling points, \n", + "the points produced by the optimization runs, and the local Minima.\n", "\n", + "This may take up to about 30 seconds to produce the 3D animation, depending on system." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.animation as animation\n", + "from IPython.display import HTML\n", + "from matplotlib.lines import Line2D\n", + "\n", + "def animate_aposmm_3d(H, batch_size):\n", + " x_vals = np.linspace(-2, 2, 50)\n", + " y_vals = np.linspace(-1, 1.1, 50)\n", + " X, Y = np.meshgrid(x_vals, y_vals)\n", + " Z = np.array([six_hump_camel_func(np.array([x, y])) for x, y in zip(X.ravel(), Y.ravel())]).reshape(X.shape)\n", + " fig = plt.figure(figsize=(10, 8))\n", + " ax = fig.add_subplot(111, projection=\"3d\")\n", + " ax.plot_surface(X, Y, Z, cmap=\"winter\", edgecolor='k', linewidth=0.1, antialiased=True, alpha=0.5) \n", + " sc_normal = ax.scatter3D([], [], [], s=6, color=\"black\", marker=\"o\", label=\"Point\")\n", + " sc_localp = ax.scatter3D([], [], [], s=40, color=\"red\", marker=\"^\", label=\"Optimization point\")\n", + " custom_M_marker = Line2D([0], [0], linestyle='None', marker='$\\\\mathrm{M}$',\n", + " markersize=8, markerfacecolor='black', markeredgecolor='black', color='white')\n", + " ax.legend([sc_normal, sc_localp, custom_M_marker], [\"Point\", \"Optimization point\", \"Local minimum\"],loc=\"upper left\")\n", + " fig.tight_layout()\n", + " annotations = []\n", + "\n", + " def update(frame):\n", + " for ann in annotations:\n", + " ann.remove()\n", + " annotations.clear()\n", + " end = min((frame + 1) * batch_size, len(H))\n", + " H_sub = H[:end]\n", + " masks = [~H_sub[\"local_pt\"] & ~H_sub[\"local_min\"], H_sub[\"local_pt\"], H_sub[\"local_min\"]]\n", + " (x_n, y_n, f_n), (x_lp, y_lp, f_lp), (x_lm, y_lm, f_lm) = [\n", + " (H_sub[\"x\"][m, 0], H_sub[\"x\"][m, 1], H_sub[\"f\"][m]) for m in masks\n", + " ]\n", + " sc_normal._offsets3d = (x_n, y_n, f_n)\n", + " sc_localp._offsets3d = (x_lp, y_lp, f_lp)\n", + " for i in range(len(x_lm)):\n", + " annotations.append(ax.text(x_lm[i], y_lm[i], f_lm[i], \"M\", color=\"white\", fontsize=12,\n", + " bbox=dict(facecolor=\"black\", alpha=0.7, pad=2), zorder=999\n", + " ))\n", + " return sc_normal, sc_localp\n", + " total_frames = (len(H) + batch_size - 1) // batch_size\n", + " ani = animation.FuncAnimation(fig, update, frames=total_frames, interval=500, blit=False, repeat=False)\n", + " plt.close(fig)\n", + " return HTML(ani.to_jshtml())\n", + "\n", + "# Reduce batch_size for more refined steps\n", + "animate_aposmm_3d(H, batch_size=50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "## Applications\n", "\n", "APOSMM is not limited to evaluating minima from pure Python simulation functions.\n", "Many common libEnsemble use-cases involve using libEnsemble's Executor to launch user\n", "applications with parameters requested by APOSMM, then evaluate their output using\n", "APOSMM, and repeat until minima are identified. A currently supported example\n", - "can be found in libEnsemble's [WarpX Scaling Test](https://github.com/Libensemble/libensemble/tree/main/libensemble/tests/scaling_tests/warpx)" + "can be found in libEnsemble's [WarpX Scaling Test](https://github.com/Libensemble/libensemble/tree/main/libensemble/tests/scaling_tests/warpx)\n", + "\n", + "Please see the [API reference](https://libensemble.readthedocs.io/en/main/examples/aposmm.html) for more APOSMM configuration options and other information." ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", + "language": "python", "name": "python3" }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/examples/tutorials/forces_with_executor/forces_tutorial_notebook.ipynb b/examples/tutorials/forces_with_executor/forces_tutorial_notebook.ipynb index bf408258a4..b85222e445 100644 --- a/examples/tutorials/forces_with_executor/forces_tutorial_notebook.ipynb +++ b/examples/tutorials/forces_with_executor/forces_tutorial_notebook.ipynb @@ -587,9 +587,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "! ls -l ensemble/sim*" diff --git a/examples/tutorials/gpcam_surrogate_model/gpcam.ipynb b/examples/tutorials/gpcam_surrogate_model/gpcam.ipynb new file mode 100644 index 0000000000..29616f582e --- /dev/null +++ b/examples/tutorials/gpcam_surrogate_model/gpcam.ipynb @@ -0,0 +1,447 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Surrogate Model with gpCAM \n", + "\n", + "This example uses [gpCAM](https://github.com/lbl-camera/gpCAM) to construct a global surrogate of ``f`` values using a Gaussian process.\n", + "\n", + "In each iteration, a batch of points is produced for concurrent evaluation, maximizing uncertainty reduction." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Before Beginning\n", + "\n", + "Ensure that libEnsemble, and gpCAM are installed via: `pip install libensemble gpcam`\n", + " \n", + "> **Note that for notebooks** the multiprocessing start method should be set to `fork` (default on Linux).\n", + "> To use with `spawn` (default on Windows and macOS), use the `multiprocess` library.\n", + "\n", + "> **Note:** If using **Colab** the cell below installs gpCAM and prevents Colab downgrading numpy due to pre-installs.\n", + "> Restart session when prompted (the warnings can be ignored)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "if 'google.colab' in sys.modules:\n", + " !pip install libensemble\n", + " # !pip install gpcam\n", + " # Prevent downgraded numpy in colab due to preinstalls\n", + " !pip install --upgrade --force-reinstall numpy==2.1.1 scipy gpcam fvgp\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generator function\n", + "\n", + "The gpCAM generator function is called ``persistent_gpCAM``.\n", + "\n", + "This persistent generator is started at the beginning and runs until the Ensemble closes down.\n", + "\n", + "This is a version of the gpCAM generator that can be found, along with other gpCAM generator functions, at [libensemble/gen_funcs/persistent_gpCAM.py](https://github.com/Libensemble/libensemble/blob/main/libensemble/gen_funcs/persistent_gpCAM.py) and can be imported from that location when libEnsemble is installed as follows:\n", + "\n", + "``from libensemble.gen_funcs.persistent_gpCAM import persistent_gpCAM``" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from numpy.lib.recfunctions import repack_fields\n", + "from gpcam import GPOptimizer as GP\n", + "\n", + "# Standard options for manager/generator comms\n", + "from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG\n", + "from libensemble.tools.persistent_support import PersistentSupport\n", + "\n", + "def persistent_gpCAM(H_in, persis_info, gen_specs, libE_info):\n", + " \"\"\"Run a batched gpCAM model to create a surrogate\"\"\"\n", + "\n", + " # Initialize\n", + " rng, batch_size, n, lb, ub, x_new, y_new, ps = _initialize_gpcAM(gen_specs[\"user\"], libE_info)\n", + " ask_max_iter = gen_specs[\"user\"].get(\"ask_max_iter\") or 10\n", + " test_points = _read_testpoints(gen_specs[\"user\"])\n", + " noise = 1e-8 # Initializes noise\n", + " my_gp = None\n", + " \n", + " # Start with a batch of random points\n", + " x_new = rng.uniform(lb, ub, (batch_size, n))\n", + " H_o = np.zeros(batch_size, dtype=gen_specs[\"out\"]) \n", + " H_o[\"x\"] = x_new\n", + " tag, Work, calc_in = ps.send_recv(H_o) # Send random points for evaluation and wait for results\n", + "\n", + " while tag not in [STOP_TAG, PERSIS_STOP]:\n", + " y_new = np.atleast_2d(calc_in[\"f\"]).T\n", + " my_gp = _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise)\n", + "\n", + " # Request new points\n", + " x_new = my_gp.ask(\n", + " input_set=np.column_stack((lb, ub)),\n", + " n=batch_size,\n", + " pop_size=batch_size,\n", + " acquisition_function=\"total correlation\",\n", + " max_iter=ask_max_iter, # Larger takes longer. gpCAM default is 20.\n", + " )[\"x\"]\n", + " \n", + " H_o = np.zeros(batch_size, dtype=gen_specs[\"out\"])\n", + " H_o[\"x\"] = x_new\n", + " tag, Work, calc_in = ps.send_recv(H_o) # Send points for evaluation and wait for results\n", + "\n", + " # If final points were returned update the model\n", + " if calc_in is not None:\n", + " y_new = np.atleast_2d(calc_in[\"f\"]).T\n", + " my_gp = _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise)\n", + "\n", + " return None, persis_info, FINISHED_PERSISTENT_GEN_TAG" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Common acquisition functions include:\n", + "\n", + "**Uncertainty reduction:**\n", + "- **\"variance\"** (default): The optimizer will find N best points. \n", + "- **\"total correlation\"**: More expensive but points found are self-avoiding.\n", + "\n", + "**Bayesian optimization:**\n", + "\n", + "These produce one point at a time unless using the [HGDL](https://ieeexplore.ieee.org/abstract/document/9652812) option.\n", + "- **\"ucb\" / \"lcb\"**: Upper/Lower Confidence Bound.\n", + "- **\"expected improvement\"**: Expected Improvement.\n", + "\n", + "For more options see: https://gpcam.lbl.gov/examples/acquisition-functions\n", + "\n", + "---\n", + "\n", + "The following cell adds the functions used by `persistent_gpCAM`.\n", + "\n", + "`_update_gp` is where the GP is fed the data and trained. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def _initialize_gpcAM(user_specs, libE_info):\n", + " \"\"\"Extract user params\"\"\"\n", + " rng_seed = user_specs.get(\"rng_seed\") # will default to None\n", + " rng = np.random.default_rng(rng_seed) # Create random stream\n", + " b = user_specs[\"batch_size\"]\n", + " lb = np.array(user_specs[\"lb\"])\n", + " ub = np.array(user_specs[\"ub\"])\n", + " n = len(lb) # no. of dimensions\n", + " init_x = np.empty((0, n))\n", + " init_y = np.empty((0, 1))\n", + " ps = PersistentSupport(libE_info, EVAL_GEN_TAG) # init comms\n", + " return rng, b, n, lb, ub, init_x, init_y, ps\n", + "\n", + "\n", + "def _read_testpoints(U):\n", + " \"\"\"Read numpy file containing evaluated points for measuring GP error\"\"\"\n", + " test_points_file = U.get(\"test_points_file\")\n", + " if test_points_file is None:\n", + " return None\n", + " test_points = np.load(test_points_file)\n", + " test_points = repack_fields(test_points[[\"x\", \"f\"]])\n", + " return test_points\n", + "\n", + "\n", + "def _compare_testpoints(my_gp, test_points, persis_info):\n", + " \"\"\"Compare model at test points\"\"\"\n", + " if test_points is None:\n", + " return\n", + " f_est = my_gp.posterior_mean(test_points[\"x\"])[\"f(x)\"]\n", + " mse = np.mean((f_est - test_points[\"f\"]) ** 2)\n", + " persis_info.setdefault(\"mean_squared_error\", []).append(float(mse))\n", + "\n", + "\n", + "def _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise):\n", + " \"\"\"Update Gaussian process with new points and train\"\"\"\n", + " noise_arr = noise * np.ones(len(y_new)) # Initializes noise\n", + " if my_gp is None:\n", + " my_gp = GP(x_new, y_new.flatten(), noise_variances=noise_arr)\n", + " else:\n", + " my_gp.tell(x_new, y_new.flatten(), noise_variances=noise_arr, append=True)\n", + " my_gp.train()\n", + " \n", + " if test_points is not None:\n", + " _compare_testpoints(my_gp, test_points, persis_info)\n", + "\n", + " return my_gp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simulator function\n", + "\n", + "Simulator functions or `sim_f`s perform calculations based on parameters created in the generator function.\n", + "Each worker will run a copy of this function in parallel.\n", + "\n", + "The function used here is the simple 2D `six_hump_camel`, for demonstration purposes.\n", + "\n", + "For running parallel applications in the simulator see the [forces examples](https://github.com/Libensemble/libensemble/tree/main/libensemble/tests/scaling_tests/forces/forces_simple)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define our simulation function\n", + "import numpy as np\n", + "\n", + "def six_hump_camel(H, persis_info, sim_specs, _):\n", + " \"\"\"Six-Hump Camel sim_f.\"\"\"\n", + " \n", + " batch = len(H[\"x\"]) # Num evaluations each sim_f call.\n", + " H_o = np.zeros(batch, dtype=sim_specs[\"out\"]) # Define output array H\n", + " \n", + " for i, x in enumerate(H[\"x\"]):\n", + " H_o[\"f\"][i] = six_hump_camel_func(x) # Function evaluations placed into H\n", + " \n", + " return H_o, persis_info\n", + "\n", + "\n", + "def six_hump_camel_func(x):\n", + " \"\"\"Six-Hump Camel function definition\"\"\"\n", + " x1 = x[0]\n", + " x2 = x[1]\n", + " term1 = (4 - 2.1 * x1**2 + (x1**4) / 3) * x1**2\n", + " term2 = x1 * x2\n", + " term3 = (-4 + 4 * x2**2) * x2**2\n", + " \n", + " return term1 + term2 + term3" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Calling Script\n", + "\n", + "Our calling script contains the configuration for libEnsemble, the generator function, and the simulator function. We then create the ensemble object and are ready to run the ensemble.\n", + "\n", + "First we will create a cleanup script so we can easily re-run." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# To rerun this notebook, we need to delete the ensemble directory.\n", + "import shutil\n", + "def cleanup():\n", + " try:\n", + " shutil.rmtree(\"ensemble\")\n", + " except:\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from pprint import pprint\n", + "\n", + "from libensemble import Ensemble\n", + "from libensemble.specs import LibeSpecs, GenSpecs, SimSpecs, AllocSpecs, ExitCriteria \n", + "\n", + "# If importing from libensemble\n", + "# from libensemble.gen_funcs.persistent_gpCAM import persistent_gpCAM\n", + "# from libensemble.sim_funcs.six_hump_camel import six_hump_camel\n", + "from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens\n", + "import warnings\n", + "\n", + "\n", + "warnings.filterwarnings(\"ignore\", message=\"Default hyperparameter_bounds\")\n", + "warnings.filterwarnings(\"ignore\", message=\"Hyperparameters initialized\")\n", + "\n", + "nworkers = 4\n", + "\n", + "# When using gen_on_manager, nworkers is number of concurrent sims.\n", + "# final_gen_send means the last evaluated points are returned to the generator to update the model.\n", + "libE_specs = LibeSpecs(nworkers=nworkers, gen_on_manager=True, final_gen_send=True)\n", + "\n", + "n = 2 # Input dimensions\n", + "batch_size = 4\n", + "num_batches = 6\n", + "\n", + "gen_specs = GenSpecs(\n", + " gen_f=persistent_gpCAM, # Generator function\n", + " persis_in=[\"f\"], # Objective, defined in sim, is returned to gen\n", + " outputs=[(\"x\", float, (n,))], # Parameters (name, type, size)\n", + " user={\n", + " \"batch_size\": batch_size,\n", + " \"lb\": np.array([-2, -1]), # lower boundaries for n dimensions\n", + " \"ub\": np.array([2, 1]), # upper boundaries for n dimensions \n", + " \"ask_max_iter\": 5, # Number of iterations for ask\n", + " \"rng_seed\": 0,\n", + " },\n", + ")\n", + "\n", + "sim_specs = SimSpecs(\n", + " sim_f=six_hump_camel, # Simulator function\n", + " inputs=[\"x\"], # Input field names. \"x\" defined in gen\n", + " outputs=[(\"f\", float)], # Objective\n", + ")\n", + "\n", + "# Starts one persistent generator. Simulated values are returned in batch.\n", + "alloc_specs = AllocSpecs(\n", + " alloc_f=only_persistent_gens,\n", + " user={\"async_return\": False}, # False = batch returns\n", + ")\n", + "\n", + "exit_criteria = ExitCriteria(sim_max=num_batches*batch_size)\n", + "\n", + "# Initialize and run the ensemble.\n", + "ensemble = Ensemble(\n", + " libE_specs=libE_specs,\n", + " sim_specs=sim_specs,\n", + " gen_specs=gen_specs,\n", + " alloc_specs=alloc_specs,\n", + " exit_criteria=exit_criteria,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At the end of our calling script we run the ensemble. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# To ensure re-running works - clean output and reset any persistent information\n", + "cleanup()\n", + "ensemble.persis_info = {}\n", + "\n", + "H, persis_info, flag = ensemble.run() # Start the ensemble. Blocks until completion.\n", + "ensemble.save_output(\"H_array\", append_attrs=False) # Save H (history of all evaluated points) to file\n", + "pprint(H[[\"sim_id\", \"x\", \"f\"]][:16]) # See first 16 results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Rerun and test model at known points\n", + "\n", + "To see how our model improves, we can use our existing points as test points and run again with a different seed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ensemble.gen_specs.user[\"rng_seed\"] = 123\n", + "ensemble.gen_specs.user[\"test_points_file\"] = \"H_array.npy\" # our previous file\n", + "\n", + "# To ensure re-running works - clean output and reset any persistent information\n", + "cleanup()\n", + "ensemble.persis_info = {}\n", + "\n", + "H, persis_info, flag = ensemble.run()\n", + "print(persis_info)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Viewing model progression\n", + "Now we can check how our model compared against the known test points at each iteration.\n", + "The comparison is based on the ***Mean Squared Error*** between the gpCAM model and our known\n", + "values at the test points.\n", + "\n", + "> **Note:** The graph may differ between runs because, although we seed libEnsemble's random number generator, \n", + "> gpCAM introduces some randomness when initializing hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Get \"mean_squared_error\" from generators return (worker 0 as we ran gen_on_manager)\n", + "mse = persis_info[0][\"mean_squared_error\"]\n", + "niter = len(mse)\n", + "num_sims = list(range(batch_size, (niter * batch_size) + 1, batch_size))\n", + "\n", + "# Plotting the data\n", + "markersize = 10\n", + "plt.figure(figsize=(10, 5))\n", + "plt.plot(\n", + " num_sims, mse, marker=\"^\", markeredgecolor=\"black\", markeredgewidth=2, \n", + " markersize=markersize, linewidth=2, label=\"Mean squared error\"\n", + ")\n", + "plt.xticks(num_sims)\n", + "\n", + "# Labeling the axes and the legend\n", + "plt.title('Mean Squared Error at test points')\n", + "plt.xlabel(\"Number of simulations\")\n", + "plt.ylabel('Mean squared error (rad$^2$)')\n", + "legend = plt.legend(framealpha=1, edgecolor=\"black\") # Increase edge width here\n", + "plt.grid(True)\n", + "plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/install/gen_deps_environment.yml b/install/gen_deps_environment.yml index 9c5492663f..0d3b47aa25 100644 --- a/install/gen_deps_environment.yml +++ b/install/gen_deps_environment.yml @@ -12,3 +12,4 @@ dependencies: - mumps-mpi - DFO-LS - mpmath + - ax-platform>=0.5.0 diff --git a/install/misc_feature_requirements.txt b/install/misc_feature_requirements.txt index a8e91cecac..626b2bdefd 100644 --- a/install/misc_feature_requirements.txt +++ b/install/misc_feature_requirements.txt @@ -1 +1 @@ -globus-compute-sdk==2.33.0 +globus-compute-sdk==3.3.1 diff --git a/install/testing_requirements.txt b/install/testing_requirements.txt index ae6b37de0b..935926ca7b 100644 --- a/install/testing_requirements.txt +++ b/install/testing_requirements.txt @@ -1,11 +1,11 @@ -flake8==7.1.1 +flake8==7.2.0 coverage>=7.5 -pytest==8.3.4 -pytest-cov==6.0.0 +pytest==8.3.5 +pytest-cov==6.1.1 pytest-timeout==2.3.1 -mock==5.1.0 +mock==5.2.0 python-dateutil==2.9.0.post0 -anyio==4.7.0 -matplotlib==3.9.3 +anyio==4.9.0 +matplotlib==3.10.1 mpmath==1.3.0 -rich==13.9.4 +rich==14.0.0 diff --git a/install/ubuntu_no312.txt b/install/ubuntu_no312.txt deleted file mode 100644 index 35964ae6cc..0000000000 --- a/install/ubuntu_no312.txt +++ /dev/null @@ -1,4 +0,0 @@ -gpcam==8.1.6 -scikit-build==0.18.1 -packaging==24.1 -git+https://github.com/bandframework/surmise.git diff --git a/libensemble/alloc_funcs/give_sim_work_first.py b/libensemble/alloc_funcs/give_sim_work_first.py index 1e528917b6..f001006f3a 100644 --- a/libensemble/alloc_funcs/give_sim_work_first.py +++ b/libensemble/alloc_funcs/give_sim_work_first.py @@ -26,7 +26,7 @@ def give_sim_work_first( work is given out unless all entries in ``H`` are returned. Can give points in highest priority, if ``"priority"`` is a field in ``H``. - If alloc_specs["user"]["give_all_with_same_priority"] is set to True, then + If ``alloc_specs["user"]["give_all_with_same_priority"]`` is set to True, then all points with the same priority value are given as a batch to the sim. Workers performing sims will be assigned resources given in H["resource_sets"] diff --git a/libensemble/alloc_funcs/start_only_persistent.py b/libensemble/alloc_funcs/start_only_persistent.py index 101a90f516..7781ed3b5f 100644 --- a/libensemble/alloc_funcs/start_only_persistent.py +++ b/libensemble/alloc_funcs/start_only_persistent.py @@ -34,6 +34,10 @@ def only_persistent_gens(W, H, sim_specs, gen_specs, alloc_specs, persis_info, l async_return: Boolean, optional Return results to gen as they come in (after sample). Default: False (batch return). + give_all_with_same_priority: Boolean, optional + If True, then all points with the same priority value are given as a batch to the sim. + Default is False + active_recv_gen: Boolean, optional Create gen in active receive mode. If True, the manager does not need to wait for a return from the generator before sending further returned points. diff --git a/libensemble/ensemble.py b/libensemble/ensemble.py index 31549d5b53..9b581a1d1a 100644 --- a/libensemble/ensemble.py +++ b/libensemble/ensemble.py @@ -563,17 +563,29 @@ def add_random_streams(self, num_streams: int = 0, seed: str = ""): self.persis_info = add_unique_random_streams(self.persis_info, nstreams + 1, seed=seed) return self.persis_info - def save_output(self, file: str): + def save_output(self, basename: str, append_attrs: bool = True): """ Writes out History array and persis_info to files. If using a workflow_dir, will place with specified filename in that directory. - Format: ``_results_History_length=_evals=_ranks=`` + Parameters + ---------- + + Format: ``_results_History_length=_evals=_ranks=`` + + To have the filename be only the basename, set append_attrs=False + + Format: ``_results_History_length=_evals=_ranks=`` """ if self.is_manager: if self._get_option("libE_specs", "workflow_dir_path"): save_libE_output( - self.H, self.persis_info, file, self.nworkers, dest_path=self.libE_specs.workflow_dir_path + self.H, + self.persis_info, + basename, + self.nworkers, + dest_path=self.libE_specs.workflow_dir_path, + append_attrs=append_attrs, ) else: - save_libE_output(self.H, self.persis_info, file, self.nworkers) + save_libE_output(self.H, self.persis_info, basename, self.nworkers, append_attrs=append_attrs) diff --git a/libensemble/executors/mpi_executor.py b/libensemble/executors/mpi_executor.py index 96c4d0d090..28d1fb6f97 100644 --- a/libensemble/executors/mpi_executor.py +++ b/libensemble/executors/mpi_executor.py @@ -47,29 +47,32 @@ class MPIExecutor(Executor): information using the ``custom_info`` argument. This takes a dictionary of values. - The allowable fields are:: + The allowable fields are: - 'mpi_runner' [string]: - Select runner: 'mpich', 'openmpi', 'aprun', 'srun', 'jsrun', 'custom' - All except 'custom' relate to runner classes in libEnsemble. + .. parsed-literal:: + + **'mpi_runner'** [string]: + Select runner: `'mpich'`, `'openmpi'`, `'aprun'`, `'srun'`, `'jsrun'`, `'custom'` + All except `'custom'` relate to runner classes in libEnsemble. Custom allows user to define their own run-lines but without parsing arguments or making use of auto-resources. - 'runner_name' [string]: - Runner name: Replaces run command if present. All runners have a default - except for 'custom'. - 'subgroup_launch' [bool]: + **'runner_name'** [string]: + The literal string that appears at the front of the run command. + This is typically 'mpirun', 'srun', etc., and can be a full path. + Defaults exist for all runners except 'custom'. + **'subgroup_launch'** [bool]: Whether MPI runs should be initiated in a new process group. This needs to be correct for kills to work correctly. Use the standalone test at - libensemble/tests/standalone_tests/kill_test to determine correct value + `libensemble/tests/standalone_tests/kill_test` to determine correct value for a system. - For example:: + For example:: - customizer = {'mpi_runner': 'mpich', - 'runner_name': 'wrapper -x mpich'} + customizer = {'mpi_runner': 'mpich', + 'runner_name': 'wrapper -x mpich'} - from libensemble.executors.mpi_executor import MPIExecutor - exctr = MPIExecutor(custom_info=customizer) + from libensemble.executors.mpi_executor import MPIExecutor + exctr = MPIExecutor(custom_info=customizer) """ @@ -336,6 +339,9 @@ def submit( else: mpi_runner_obj = self.mpi_runner_obj or self._create_mpi_runner_from_attr() + if env_script is None and mpi_runner_obj is None: + raise ExecutorException("No valid MPI runner was found") + mpi_specs = mpi_runner_obj.get_mpi_specs( task, num_procs, diff --git a/libensemble/executors/mpi_runner.py b/libensemble/executors/mpi_runner.py index eb002d14b5..48953cc3c9 100644 --- a/libensemble/executors/mpi_runner.py +++ b/libensemble/executors/mpi_runner.py @@ -21,11 +21,13 @@ def get_runner(mpi_runner_type, runner_name=None, platform_info=None): "msmpi": MSMPI_MPIRunner, "custom": MPIRunner, } - mpi_runner = mpi_runners[mpi_runner_type] - if runner_name is not None: - runner = mpi_runner(run_command=runner_name, platform_info=platform_info) - else: - runner = mpi_runner(platform_info=platform_info) + runner = None + if mpi_runner_type is not None: + mpi_runner = mpi_runners[mpi_runner_type] + if runner_name is not None: + runner = mpi_runner(run_command=runner_name, platform_info=platform_info) + else: + runner = mpi_runner(platform_info=platform_info) return runner def __init__(self, run_command="mpiexec", platform_info=None): diff --git a/libensemble/gen_funcs/aposmm_localopt_support.py b/libensemble/gen_funcs/aposmm_localopt_support.py index 0bd1b9f3c7..3545afd892 100644 --- a/libensemble/gen_funcs/aposmm_localopt_support.py +++ b/libensemble/gen_funcs/aposmm_localopt_support.py @@ -75,11 +75,11 @@ class LocalOptInterfacer(object): Currently supported routines are - - NLopt routines ['LN_SBPLX', 'LN_BOBYQA', 'LN_COBYLA', 'LN_NEWUOA', 'LN_NELDERMEAD', 'LD_MMA'] - - PETSc/TAO routines ['pounders', 'blmvm', 'nm'] - - SciPy routines ['scipy_Nelder-Mead', 'scipy_COBYLA', 'scipy_BFGS'] - - DFOLS ['dfols'] - - External local optimizer ['external_localopt'] (which use files to pass/receive x/f values) + - NLopt [``'LN_SBPLX'``, ``LN_BOBYQA'``, ``'LN_COBYLA'``, ``'LN_NEWUOA'``, ``'LN_NELDERMEAD'``, ``'LD_MMA'``] + - PETSc/TAO [``'pounders'``, ``'blmvm'``, ``'nm'``] + - SciPy [``'scipy_Nelder-Mead'``, ``'scipy_COBYLA'``, ``'scipy_BFGS'``] + - DFOLS [``'dfols'``] + - External local optimizer [``'external_localopt'``] (which use files to pass/receive ``x/f`` values) """ def __init__(self, user_specs, x0, f0, grad0=None): diff --git a/libensemble/gen_funcs/persistent_aposmm.py b/libensemble/gen_funcs/persistent_aposmm.py index c5c3aa5e65..1dc3af8e13 100644 --- a/libensemble/gen_funcs/persistent_aposmm.py +++ b/libensemble/gen_funcs/persistent_aposmm.py @@ -37,85 +37,78 @@ def cdist(XA, XB, metric="euclidean"): def aposmm(H, persis_info, gen_specs, libE_info): """ - APOSMM coordinates multiple local optimization runs, starting from points - which do not have a better point nearby (within a distance ``r_k``). This - generation function uses a ``local_H`` (serving a similar purpose as ``H`` - in libEnsemble) containing the fields: - - - ``'x' [n floats]``: Parameters being optimized over - - ``'x_on_cube' [n floats]``: Parameters scaled to the unit cube - - ``'f' [float]``: Objective function being minimized - - ``'local_pt' [bool]``: True if point from a local optimization run - - ``'dist_to_unit_bounds' [float]``: Distance to domain boundary - - ``'dist_to_better_l' [float]``: Dist to closest better local opt point - - ``'dist_to_better_s' [float]``: Dist to closest better sample point - - ``'ind_of_better_l' [int]``: Index of point ``'dist_to_better_l``' away - - ``'ind_of_better_s' [int]``: Index of point ``'dist_to_better_s``' away - - ``'started_run' [bool]``: True if point has started a local opt run - - ``'num_active_runs' [int]``: Number of active local runs point is in - - ``'local_min' [float]``: True if point has been ruled a local minima - - ``'sim_id' [int]``: Row number of entry in history + APOSMM coordinates multiple local optimization runs, dramatically reducing time for + discovering multiple minima on parallel systems. APOSMM tracks these fields: + + - ``"x" [n floats]``: Parameters being optimized over + - ``"x_on_cube" [n floats]``: Parameters scaled to the unit cube + - ``"f" [float]``: Objective function being minimized + - ``"local_pt" [bool]``: True if point from a local optimization run + - ``"started_run" [bool]``: True if point has started a local opt run + - ``"num_active_runs" [int]``: Number of active local runs point is in + - ``"local_min" [float]``: True if point has been ruled a local minima + - ``"sim_id" [int]``: Row number of entry in history and optionally - - ``'fvec' [m floats]``: All objective components (if performing a least-squares calculation) - - ``'grad' [n floats]``: The gradient (if available) of the objective with respect to `x`. + - ``"fvec" [m floats]``: All objective components (if performing a least-squares calculation) + - ``"grad" [n floats]``: The gradient (if available) of the objective with respect to `x`. Note: - If any of the above fields are desired after a libEnsemble run, name - them in ``gen_specs['out']``. + them in ``gen_specs["out"]``. - If intitializing APOSMM with past function values, make sure to include - ``'x'``, ``'x_on_cube'``, ``'f'``, ``'local_pt'``, etc. in - ``gen_specs['in']`` (and, of course, include them in the H0 array given + ``"x"``, ``"x_on_cube"``, ``"f"``, ``"local_pt"``, etc. in + ``gen_specs["in"]`` (and, of course, include them in the H0 array given to libensemble). - Necessary quantities in ``gen_specs['user']`` are: + Necessary quantities in ``gen_specs["user"]`` are: - - ``'lb' [n floats]``: Lower bound on search domain - - ``'ub' [n floats]``: Upper bound on search domain - - ``'localopt_method' [str]``: Name of an NLopt, PETSc/TAO, or SciPy method - (see 'advance_local_run' below for supported methods). When using a SciPy - method, must supply ``'opt_return_codes'``, a list of integers that will + - ``"lb" [n floats]``: Lower bound on search domain + - ``"ub" [n floats]``: Upper bound on search domain + - ``"localopt_method" [str]``: Name of an NLopt, PETSc/TAO, or SciPy method + (see "advance_local_run" below for supported methods). When using a SciPy + method, must supply ``"opt_return_codes"``, a list of integers that will be used to determine if the x produced by the localopt method should be - ruled a local minimum. (For example, SciPy's COBYLA has a 'status' of 1 if - at an optimum, but SciPy's Nelder-Mead and BFGS have a 'status' of 0 if at + ruled a local minimum. (For example, SciPy's COBYLA has a "status" of ``1`` if + at an optimum, but SciPy's Nelder-Mead and BFGS have a "status" of ``0`` if at an optimum.) - - ``'initial_sample_size' [int]``: Number of uniformly sampled points - must be returned (non-nan value) before a local opt run is started. Can be + - ``"initial_sample_size" [int]``: Number of uniformly sampled points + to be evaluated before starting the localopt runs. Can be zero if no additional sampling is desired, but if zero there must be past sim_f values given to libEnsemble in H0. - Optional ``gen_specs['user']`` entries are: + Optional ``gen_specs["user"]`` entries are: - - ``'sample_points' [numpy array]``: Points to be sampled (original domain). + - ``"max_active_runs" [int]``: Bound on number of runs APOSMM is advancing + - ``"sample_points" [numpy array]``: Points to be sampled (original domain). If more sample points are needed by APOSMM during the course of the optimization, points will be drawn uniformly over the domain - - ``'components' [int]``: Number of objective components - - ``'dist_to_bound_multiple' [float in (0, 1]]``: What fraction of the + - ``"components" [int]``: Number of objective components + - ``"dist_to_bound_multiple" [float in (0, 1]]``: What fraction of the distance to the nearest boundary should the initial step size be in localopt runs - - ``'lhs_divisions' [int]``: Number of Latin hypercube sampling partitions + - ``"lhs_divisions" [int]``: Number of Latin hypercube sampling partitions (0 or 1 results in uniform sampling) - - ``'mu' [float]``: Distance from the boundary that all localopt starting + - ``"mu" [float]``: Distance from the boundary that all localopt starting points must satisfy - - ``'nu' [float]``: Distance from identified minima that all starting + - ``"nu" [float]``: Distance from identified minima that all starting points must satisfy - - ``'rk_const' [float]``: Multiplier in front of the r_k value - - ``'max_active_runs' [int]``: Bound on number of runs APOSMM is advancing - - ``'stop_after_k_minima' [int]``: Tell APOSMM to stop after this many + - ``"rk_const" [float]``: Multiplier in front of the ``r_k`` value + - ``"stop_after_k_minima" [int]``: Tell APOSMM to stop after this many local minima have been identified by a local optimization run. - - ``'stop_after_k_runs' [int]``: Tell APOSMM to stop after this many runs + - ``"stop_after_k_runs" [int]``: Tell APOSMM to stop after this many runs have ended. (The number of ended runs may be less than the number of minima if, for example, a local optimization run ends due to a evaluation constraint, but not convergence criteria.) If the rules in ``decide_where_to_start_localopt`` produces more than - ``'max_active_runs'`` in some iteration, then existing runs are prioritized. + ``"max_active_runs"`` in some iteration, then existing runs are prioritized. - And ``gen_specs['user']`` must also contain fields for the given - localopt_method's convergence tolerances (e.g., gatol/grtol for PETSC/TAO - or ftol_rel for NLopt) + And ``gen_specs["user"]`` must also contain fields for the given + localopt_method's convergence tolerances (e.g., ``gatol/grtol`` for PETSC/TAO + or ``ftol_rel`` for NLopt) .. seealso:: diff --git a/libensemble/gen_funcs/persistent_ax_multitask.py b/libensemble/gen_funcs/persistent_ax_multitask.py index 06c3ea7bbd..451f14ad9a 100644 --- a/libensemble/gen_funcs/persistent_ax_multitask.py +++ b/libensemble/gen_funcs/persistent_ax_multitask.py @@ -8,16 +8,27 @@ This `gen_f` is meant to be used with the `alloc_f` function `only_persistent_gens` -This test currently requires ax-platform<=0.4.0 +Requires: Ax>=0.5.0 + +Ax notes: +Each arm = a set of simulation inputs (a sim_id) +Each trial = a batch of simulations. +The metric = the recorded simulation output (f) that Ax optimizes. +Ax runner handles the execution of trials - AxRunner wraps Runner to use libE tr.run() + """ import os +import warnings from copy import deepcopy +from typing import Optional import numpy as np import pandas as pd +import torch from ax import Metric, Runner from ax.core.data import Data +from ax.core.experiment import Experiment from ax.core.generator_run import GeneratorRun from ax.core.multi_type_experiment import MultiTypeExperiment from ax.core.objective import Objective @@ -25,22 +36,116 @@ from ax.core.optimization_config import OptimizationConfig from ax.core.parameter import ParameterType, RangeParameter from ax.core.search_space import SearchSpace +from pyre_extensions import assert_is_instance + +# Necessary for docs rendering (as Ax is mocked) +try: + from ax.exceptions.core import AxParameterWarning +except ImportError: + AxParameterWarning = Warning + from ax.modelbridge.factory import get_sobol +from ax.modelbridge.registry import MBM_X_trans, Models, ST_MTGP_trans +from ax.modelbridge.torch import TorchModelBridge +from ax.modelbridge.transforms.convert_metric_names import ConvertMetricNames, tconfig_from_mt_experiment +from ax.modelbridge.transforms.derelativize import Derelativize +from ax.modelbridge.transforms.stratified_standardize_y import StratifiedStandardizeY +from ax.modelbridge.transforms.task_encode import TaskChoiceToIntTaskChoice +from ax.modelbridge.transforms.trial_as_task import TrialAsTask from ax.runners import SyntheticRunner from ax.storage.json_store.save import save_experiment -from ax.storage.metric_registry import register_metric +from ax.storage.metric_registry import register_metrics from ax.storage.runner_registry import register_runner from ax.utils.common.result import Ok -try: - from ax.modelbridge.factory import get_MTGP -except ImportError: - # For Ax >= 0.3.4 - from ax.modelbridge.factory import get_MTGP_LEGACY as get_MTGP - from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG from libensemble.tools.persistent_support import PersistentSupport +__all__ = ["persistent_gp_mt_ax_gen_f"] + +warnings.filterwarnings( + "ignore", + message="`cache_root` is only supported for GPyTorchModels", + category=RuntimeWarning, +) + +warnings.filterwarnings( + "ignore", + message="Changing `is_ordered` to `True` for `ChoiceParameter`", + category=AxParameterWarning, +) + +MT_MTGP_trans = list(MBM_X_trans) + [ + Derelativize, + ConvertMetricNames, + TrialAsTask, + StratifiedStandardizeY, + TaskChoiceToIntTaskChoice, +] + + +# get_MTGP based on https://ax.dev/docs/tutorials/multi_task/ +def get_MTGP( + experiment: Experiment, + data: Data, + search_space: Optional[SearchSpace] = None, + trial_index: Optional[int] = None, + device: torch.device = torch.device("cpu"), + dtype: torch.dtype = torch.double, +) -> TorchModelBridge: + """Instantiates a Multi-task Gaussian Process (MTGP) model that generates + points with EI. + + If the input experiment is a MultiTypeExperiment then a + Multi-type Multi-task GP model will be instantiated. + Otherwise, the model will be a Single-type Multi-task GP. + """ + + if isinstance(experiment, MultiTypeExperiment): + trial_index_to_type = { + t.index: t.trial_type for t in experiment.trials.values() + } + transforms = MT_MTGP_trans + transform_configs = { + "TrialAsTask": {"trial_level_map": {"trial_type": trial_index_to_type}}, + "ConvertMetricNames": tconfig_from_mt_experiment(experiment), + } + else: + # Set transforms for a Single-type MTGP model. + transforms = ST_MTGP_trans + transform_configs = None + + # Choose the status quo features for the experiment from the selected trial. + # If trial_index is None, we will look for a status quo from the last + # experiment trial to use as a status quo for the experiment. + if trial_index is None: + trial_index = len(experiment.trials) - 1 + elif trial_index >= len(experiment.trials): + raise ValueError("trial_index is bigger than the number of experiment trials") + + status_quo = experiment.trials[trial_index].status_quo + if status_quo is None: + status_quo_features = None + else: + status_quo_features = ObservationFeatures( + parameters=status_quo.parameters, + trial_index=trial_index, # pyre-ignore[6] + ) + + return assert_is_instance( + Models.ST_MTGP( + experiment=experiment, + search_space=search_space or experiment.search_space, + data=data, + transforms=transforms, + transform_configs=transform_configs, + torch_dtype=dtype, + torch_device=device, + status_quo_features=status_quo_features, + ), + TorchModelBridge, + ) + def persistent_gp_mt_ax_gen_f(H, persis_info, gen_specs, libE_info): """ @@ -97,6 +202,7 @@ def persistent_gp_mt_ax_gen_f(H, persis_info, gen_specs, libE_info): optimization_config=opt_config, ) + # hifi_task has been added as default but we need to add lofi task and link them. exp.add_trial_type(lofi_task, ax_runner) exp.add_tracking_metric(metric=lofi_objective, trial_type=lofi_task, canonical_name="hifi_metric") @@ -141,7 +247,7 @@ def persistent_gp_mt_ax_gen_f(H, persis_info, gen_specs, libE_info): # But launch them at low fidelity. tr = exp.new_batch_trial(trial_type=lofi_task, generator_run=gr) - tr.run() + tr.run() # Runs sims via libE (see AxRunner.run below) tr.mark_completed() tag = tr.run_metadata["tag"] if tag in [STOP_TAG, PERSIS_STOP]: @@ -157,7 +263,7 @@ def persistent_gp_mt_ax_gen_f(H, persis_info, gen_specs, libE_info): # Select max-utility points from the low fidelity batch to generate a high fidelity batch. gr = max_utility_from_GP(n=n_opt_hifi, m=m, gr=gr, hifi_task=hifi_task) tr = exp.new_batch_trial(trial_type=hifi_task, generator_run=gr) - tr.run() + tr.run() # Runs sims via libE (see AxRunner.run below) tr.mark_completed() tag = tr.run_metadata["tag"] if tag in [STOP_TAG, PERSIS_STOP]: @@ -169,7 +275,9 @@ def persistent_gp_mt_ax_gen_f(H, persis_info, gen_specs, libE_info): if not os.path.exists("model_history"): os.mkdir("model_history") # Register metric and runner in order to be able to save to json. - _, encoder_registry, decoder_registry = register_metric(AxMetric) + _, encoder_registry, decoder_registry = register_metrics( + {AxMetric: None} + ) _, encoder_registry, decoder_registry = register_runner( AxRunner, encoder_registry=encoder_registry, @@ -196,7 +304,7 @@ def persistent_gp_mt_ax_gen_f(H, persis_info, gen_specs, libE_info): # Increase iteration counter. model_iteration += 1 - return [], persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG class AxRunner(Runner): @@ -222,9 +330,8 @@ def run(self, trial): for j in range(n_param): param_array[j] = params[f"x{j}"] H_o["x"][i] = param_array - H_o["resource_sets"][i] = 1 + H_o["resource_sets"][i] = 1 # one is default but could be diff for hi/lo H_o["task"][i] = task - tag, Work, calc_in = self.ps.send_recv(H_o) trial_metadata["tag"] = tag diff --git a/libensemble/gen_funcs/persistent_fd_param_finder.py b/libensemble/gen_funcs/persistent_fd_param_finder.py index 87881422e6..d72f64537a 100644 --- a/libensemble/gen_funcs/persistent_fd_param_finder.py +++ b/libensemble/gen_funcs/persistent_fd_param_finder.py @@ -35,7 +35,7 @@ def build_H0(x_f_pairs, gen_specs, noise_h_mat): def fd_param_finder(H, persis_info, gen_specs, libE_info): """ This generation function loops through a set of suitable finite difference - parameters for a mapping F from R^n to R^m. + parameters for a mapping ``F`` from ``R^n`` to ``R^m``. .. seealso:: `test_persistent_fd_param_finder.py` `_ # noqa @@ -56,16 +56,8 @@ def fd_param_finder(H, persis_info, gen_specs, libE_info): tag = None # # Request evaluations of the base point x0 at all p f_inds - # H0 = np.zeros(p, dtype=gen_specs["out"]) - # for j in range(p): - # H0["x"][j] = x0 - # H0["x_ind"][j] = -1 # Marking these to know they are the basepoint - # H0["f_ind"][j] = j - # H0["n_ind"][j] = nf/2 - # tag, Work, calc_in = sendrecv_mgr_worker_msg(comm, H0) for i in range(n): for j in range(p): - # Fhist0[i, j, nf//2] = calc_in["f_val"][calc_in["f_ind"]==j] Fhist0[i, j, nf // 2] = U["f0"][j] x_f_pairs = np.array(np.meshgrid(range(n), range(p))).T.reshape(-1, n) @@ -75,9 +67,6 @@ def fd_param_finder(H, persis_info, gen_specs, libE_info): tag, Work, calc_in = ps.send_recv(H0) - # import matlab.engine - # eng = matlab.engine.start_matlab() - # Send nf points for each (x_ind, f_ind) pair while tag not in [STOP_TAG, PERSIS_STOP]: x_f_pairs = np.unique(calc_in[["x_ind", "f_ind"]]) @@ -90,13 +79,6 @@ def fd_param_finder(H, persis_info, gen_specs, libE_info): logical_conds = (calc_in["x_ind"] == i, calc_in["f_ind"] == j, calc_in["n_ind"] == k) Fhist0[i, j, k] = calc_in["f_val"][np.logical_and.reduce(logical_conds)][0] - # Compute noise for (i, j): - # [Fnoise(i, j), ~, inform(i, j)] = ECnoise(nf-1, Fhist0(i, j, 2:nf)); - # t = eng.ECnoise(nf+1, matlab.double(Fhist0[i, j, :nf+1]), nargout=3) - # # Optional: check to see what would get with 2 fewer evals (requires nf>=4): - # [Fnoise2(i, j), ~, inform2(i, j)] = ECnoise(nf-1, Fhist0(i, j, 2:nf)); - - # cmd = ["/home/jlarson/software/MATLAB/R2019a/bin/matlab", "-batch", cmd = [ "octave", "--no-window-system", diff --git a/libensemble/gen_funcs/persistent_gpCAM.py b/libensemble/gen_funcs/persistent_gpCAM.py index 953cf33cf8..05b08bb5ed 100644 --- a/libensemble/gen_funcs/persistent_gpCAM.py +++ b/libensemble/gen_funcs/persistent_gpCAM.py @@ -10,30 +10,23 @@ from libensemble.tools.persistent_support import PersistentSupport __all__ = [ - "persistent_gpCAM_simple", - "persistent_gpCAM_ask_tell", + "persistent_gpCAM", + "persistent_gpCAM_covar", ] -def _initialize_gpcAM(user_specs, libE_info): +def _initialize_gpcAM(user_specs, libE_info, persis_info): """Extract user params""" + rng_seed = user_specs.get("rng_seed") # Will default to None + rng = persis_info.get("rand_stream") or np.random.default_rng(rng_seed) b = user_specs["batch_size"] lb = np.array(user_specs["lb"]) ub = np.array(user_specs["ub"]) - n = len(lb) # dimension - assert isinstance(b, int), "Batch size must be an integer" - assert isinstance(n, int), "Dimension must be an integer" - assert isinstance(lb, np.ndarray), "lb must be a numpy array" - assert isinstance(ub, np.ndarray), "ub must be a numpy array" - - all_x = np.empty((0, n)) - all_y = np.empty((0, 1)) - - ps = PersistentSupport(libE_info, EVAL_GEN_TAG) - - np.random.seed(0) - - return b, n, lb, ub, all_x, all_y, ps + n = len(lb) # no. of dimensions + init_x = np.empty((0, n)) + init_y = np.empty((0, 1)) + ps = PersistentSupport(libE_info, EVAL_GEN_TAG) # init comms + return rng, b, n, lb, ub, init_x, init_y, ps def _read_testpoints(U): @@ -41,7 +34,6 @@ def _read_testpoints(U): test_points_file = U.get("test_points_file") if test_points_file is None: return None - test_points = np.load(test_points_file) # Remove any NaNs @@ -54,6 +46,30 @@ def _read_testpoints(U): return test_points +def _compare_testpoints(my_gp, test_points, persis_info): + """Compare model at test points""" + if test_points is None: + return + f_est = my_gp.posterior_mean(test_points["x"])["f(x)"] + mse = np.mean((f_est - test_points["f"]) ** 2) + persis_info.setdefault("mean_squared_error", []).append(float(mse)) + + +def _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise): + """Update Gaussian process with new points and train""" + noise_arr = noise * np.ones(len(y_new)) # Initializes noise + if my_gp is None: + my_gp = GP(x_new, y_new.flatten(), noise_variances=noise_arr) + else: + my_gp.tell(x_new, y_new.flatten(), noise_variances=noise_arr, append=True) + my_gp.train() + + if test_points is not None: + _compare_testpoints(my_gp, test_points, persis_info) + + return my_gp + + def _generate_mesh(lb, ub, num_points=10): """ Generate a mesh of points in n-dimensional space over a hypercube defined by lb and ub. @@ -96,9 +112,7 @@ def _eval_var(my_gp, all_x, all_y, x_for_var, test_points, persis_info): persis_info.setdefault("mean_variance", []).append(np.mean(var_vals)) if test_points is not None: - f_est = my_gp.posterior_mean(test_points["x"])["f(x)"] - mse = np.mean((f_est - test_points["f"]) ** 2) - persis_info.setdefault("mean_squared_error", []).append(mse) + _compare_testpoints(my_gp, test_points, persis_info) return np.array(var_vals) @@ -140,23 +154,83 @@ def _find_eligible_points(x_for_var, sorted_indices, r, batch_size): return np.array(eligible_points) -def persistent_gpCAM_simple(H_in, persis_info, gen_specs, libE_info): +def persistent_gpCAM(H_in, persis_info, gen_specs, libE_info): + """ + This generation function constructs a global surrogate of `f` values. It is + a batched method that produces a first batch uniformly random from (lb, ub). + On subequent iterations, it calls an optimization method to produce the next + batch of points. This optimization might be too slow (relative to the + simulation evaluation time) for some use cases. + + .. seealso:: + `test_gpCAM.py `_ + """ # noqa + + rng, batch_size, n, lb, ub, x_new, y_new, ps = _initialize_gpcAM(gen_specs["user"], libE_info, persis_info) + ask_max_iter = gen_specs["user"].get("ask_max_iter") or 10 + test_points = _read_testpoints(gen_specs["user"]) + noise = 1e-8 # Initializes noise + my_gp = None + + # Start with a batch of random points + x_new = rng.uniform(lb, ub, (batch_size, n)) + H_o = np.zeros(batch_size, dtype=gen_specs["out"]) + H_o["x"] = x_new + tag, Work, calc_in = ps.send_recv(H_o) # Send random points for evaluation and wait + + while tag not in [STOP_TAG, PERSIS_STOP]: + y_new = np.atleast_2d(calc_in["f"]).T + my_gp = _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise) + + # Request new points + start = time.time() + x_new = my_gp.ask( + input_set=np.column_stack((lb, ub)), + n=batch_size, + pop_size=batch_size, + acquisition_function="total correlation", + max_iter=ask_max_iter, # Larger takes longer. gpCAM default is 20. + )["x"] + print(f"Ask time:{time.time() - start}") + + H_o = np.zeros(batch_size, dtype=gen_specs["out"]) + H_o["x"] = x_new + tag, Work, calc_in = ps.send_recv(H_o) + + # If final points were returned update the model + if calc_in is not None: + y_new = np.atleast_2d(calc_in["f"]).T + my_gp = _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise) + + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG + + +def persistent_gpCAM_covar(H_in, persis_info, gen_specs, libE_info): """ This generation function constructs a global surrogate of `f` values. It is a batched method that produces a first batch uniformly random from (lb, ub) and on following iterations samples the GP posterior covariance function to find sample points. + If gen_specs["user"]["use_grid"] is set to True, the parameter space is + divided into a mesh of candidate points (num_points in each dimension). + Subsequent points are chosen with maximum covariance that are at least a + distance `r` away from each other to explore difference regions. + + If gen_specs["user"]["test_points_file"] is set to a file of evaluated + points, then the gpCAM predications are compared at these points to assess + model quality. + .. seealso:: `test_gpCAM.py `_ """ # noqa + U = gen_specs["user"] my_gp = None noise = 1e-12 test_points = _read_testpoints(U) - - batch_size, n, lb, ub, all_x, all_y, ps = _initialize_gpcAM(U, libE_info) + rng, batch_size, n, lb, ub, x_new, y_new, ps = _initialize_gpcAM(gen_specs["user"], libE_info, persis_info) # Send batches until manager sends stop tag tag = None @@ -167,14 +241,14 @@ def persistent_gpCAM_simple(H_in, persis_info, gen_specs, libE_info): x_for_var = _generate_mesh(lb, ub, num_points) r_low_init, r_high_init = _calculate_grid_distances(lb, ub, num_points) else: - x_for_var = persis_info["rand_stream"].uniform(lb, ub, (10 * batch_size, n)) + x_for_var = rng.uniform(lb, ub, (10 * batch_size, n)) while tag not in [STOP_TAG, PERSIS_STOP]: - if all_x.shape[0] == 0: - x_new = persis_info["rand_stream"].uniform(lb, ub, (batch_size, n)) + if x_new.shape[0] == 0: + x_new = rng.uniform(lb, ub, (batch_size, n)) else: if not U.get("use_grid"): - x_for_var = persis_info["rand_stream"].uniform(lb, ub, (10 * batch_size, n)) + x_for_var = rng.uniform(lb, ub, (10 * batch_size, n)) x_new = x_for_var[np.argsort(var_vals)[-batch_size:]] else: r_high = r_high_init @@ -199,69 +273,10 @@ def persistent_gpCAM_simple(H_in, persis_info, gen_specs, libE_info): nan_indices = [i for i, fval in enumerate(y_new) if np.isnan(fval)] x_new = np.delete(x_new, nan_indices, axis=0) y_new = np.delete(y_new, nan_indices, axis=0) - all_x = np.vstack((all_x, x_new)) - all_y = np.vstack((all_y, y_new)) - - if my_gp is None: - my_gp = GP(all_x, all_y.flatten(), noise_variances=noise * np.ones(len(all_y))) - else: - my_gp.tell(all_x, all_y.flatten(), noise_variances=noise * np.ones(len(all_y))) - my_gp.train() + my_gp = _update_gp(my_gp, x_new, y_new, test_points, persis_info, noise) if not U.get("use_grid"): - x_for_var = persis_info["rand_stream"].uniform(lb, ub, (10 * batch_size, n)) - var_vals = _eval_var(my_gp, all_x, all_y, x_for_var, test_points, persis_info) - - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG - - -def persistent_gpCAM_ask_tell(H_in, persis_info, gen_specs, libE_info): - """ - Like persistent_gpCAM_simple, this generation function constructs a global - surrogate of `f` values. It also aa batched method that produces a first batch - uniformly random from (lb, ub). On subequent iterations, it calls an - optimization method to produce the next batch of points. This optimization - might be too slow (relative to the simulation evaluation time) for some use cases. - - .. seealso:: - `test_gpCAM.py `_ - """ # noqa - - batch_size, n, lb, ub, all_x, all_y, ps = _initialize_gpcAM(gen_specs["user"], libE_info) - ask_max_iter = gen_specs["user"].get("ask_max_iter") or 10 - - H_o = np.zeros(batch_size, dtype=gen_specs["out"]) - x_new = persis_info["rand_stream"].uniform(lb, ub, (batch_size, n)) - H_o["x"] = x_new - - tag, Work, calc_in = ps.send_recv(H_o) - - first_call = True - while tag not in [STOP_TAG, PERSIS_STOP]: - all_x = np.vstack((all_x, x_new)) - all_y = np.vstack((all_y, np.atleast_2d(calc_in["f"]).T)) - - if first_call: - # Initialize GP - my_gp = GP(all_x, all_y.flatten(), noise_variances=1e-8 * np.ones(len(all_y))) - first_call = False - else: - my_gp.tell(all_x, all_y.flatten(), noise_variances=1e-8 * np.ones(len(all_y))) - - my_gp.train() - - start = time.time() - x_new = my_gp.ask( - input_set=np.column_stack((lb, ub)), - n=batch_size, - pop_size=batch_size, - acquisition_function="total correlation", - max_iter=ask_max_iter, # Larger takes longer. gpCAM default is 20. - )["x"] - print(f"Ask time:{time.time() - start}") - H_o = np.zeros(batch_size, dtype=gen_specs["out"]) - H_o["x"] = x_new - - tag, Work, calc_in = ps.send_recv(H_o) + x_for_var = rng.uniform(lb, ub, (10 * batch_size, n)) + var_vals = _eval_var(my_gp, x_new, y_new, x_for_var, test_points, persis_info) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG diff --git a/libensemble/gen_funcs/persistent_inverse_bayes.py b/libensemble/gen_funcs/persistent_inverse_bayes.py index 4875ca03bf..2f677902b0 100644 --- a/libensemble/gen_funcs/persistent_inverse_bayes.py +++ b/libensemble/gen_funcs/persistent_inverse_bayes.py @@ -3,6 +3,10 @@ from libensemble.message_numbers import EVAL_GEN_TAG, FINISHED_PERSISTENT_GEN_TAG, PERSIS_STOP, STOP_TAG from libensemble.tools.persistent_support import PersistentSupport +__all__ = [ + "persistent_updater_after_likelihood", +] + def persistent_updater_after_likelihood(H, persis_info, gen_specs, libE_info): """ """ @@ -36,4 +40,4 @@ def persistent_updater_after_likelihood(H, persis_info, gen_specs, libE_info): if calc_in is not None: w = H_o["prior"] + calc_in["like"] - H_o["prop"] - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG diff --git a/libensemble/gen_funcs/persistent_sampling.py b/libensemble/gen_funcs/persistent_sampling.py index fcbcba0904..401ccdaa94 100644 --- a/libensemble/gen_funcs/persistent_sampling.py +++ b/libensemble/gen_funcs/persistent_sampling.py @@ -29,8 +29,8 @@ def _get_user_params(user_specs): return b, n, lb, ub -@persistent_input_fields(["f", "x", "sim_id"]) -@output_data([("x", float, (2,))]) +@persistent_input_fields(["sim_id"]) +@output_data([("x", float, (2,))]) # The dimesion of 2 is a default and can be overwritten def persistent_uniform(_, persis_info, gen_specs, libE_info): """ This generation function always enters into persistent mode and returns @@ -60,7 +60,7 @@ def persistent_uniform(_, persis_info, gen_specs, libE_info): if hasattr(calc_in, "__len__"): b = len(calc_in) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def persistent_uniform_final_update(_, persis_info, gen_specs, libE_info): @@ -163,7 +163,7 @@ def persistent_request_shutdown(_, persis_info, gen_specs, libE_info): print("Reached threshold.", f_count, flush=True) break # End the persistent gen - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def uniform_nonblocking(_, persis_info, gen_specs, libE_info): @@ -197,7 +197,7 @@ def uniform_nonblocking(_, persis_info, gen_specs, libE_info): if hasattr(calc_in, "__len__"): b = len(calc_in) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def batched_history_matching(_, persis_info, gen_specs, libE_info): @@ -243,7 +243,7 @@ def batched_history_matching(_, persis_info, gen_specs, libE_info): mu = np.mean(H_o["x"][best_inds], axis=0) Sigma = np.cov(H_o["x"][best_inds].T) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def persistent_uniform_with_cancellations(_, persis_info, gen_specs, libE_info): @@ -272,4 +272,4 @@ def persistent_uniform_with_cancellations(_, persis_info, gen_specs, libE_info): cancel_from += b ps.request_cancel_sim_ids(cancel_ids) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG diff --git a/libensemble/gen_funcs/persistent_sampling_var_resources.py b/libensemble/gen_funcs/persistent_sampling_var_resources.py index 252fe6019d..394ef75815 100644 --- a/libensemble/gen_funcs/persistent_sampling_var_resources.py +++ b/libensemble/gen_funcs/persistent_sampling_var_resources.py @@ -58,7 +58,7 @@ def uniform_sample(_, persis_info, gen_specs, libE_info): if hasattr(calc_in, "__len__"): b = len(calc_in) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def uniform_sample_with_var_gpus(_, persis_info, gen_specs, libE_info): @@ -99,7 +99,7 @@ def uniform_sample_with_var_gpus(_, persis_info, gen_specs, libE_info): if hasattr(calc_in, "__len__"): b = len(calc_in) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def uniform_sample_with_procs_gpus(_, persis_info, gen_specs, libE_info): @@ -128,7 +128,7 @@ def uniform_sample_with_procs_gpus(_, persis_info, gen_specs, libE_info): if hasattr(calc_in, "__len__"): b = len(calc_in) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def uniform_sample_with_var_priorities(_, persis_info, gen_specs, libE_info): @@ -163,7 +163,7 @@ def uniform_sample_with_var_priorities(_, persis_info, gen_specs, libE_info): tag, Work, calc_in = ps.send_recv(H_o) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def uniform_sample_diff_simulations(_, persis_info, gen_specs, libE_info): @@ -197,7 +197,7 @@ def uniform_sample_diff_simulations(_, persis_info, gen_specs, libE_info): if hasattr(calc_in, "__len__"): b = len(calc_in) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def uniform_sample_with_sim_gen_resources(_, persis_info, gen_specs, libE_info): @@ -245,4 +245,4 @@ def uniform_sample_with_sim_gen_resources(_, persis_info, gen_specs, libE_info): if hasattr(calc_in, "__len__"): b = len(calc_in) - return H_o, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG diff --git a/libensemble/gen_funcs/persistent_tasmanian.py b/libensemble/gen_funcs/persistent_tasmanian.py index cedb87c1ea..ac491cf6a0 100644 --- a/libensemble/gen_funcs/persistent_tasmanian.py +++ b/libensemble/gen_funcs/persistent_tasmanian.py @@ -1,6 +1,6 @@ """ A persistent generator using the uncertainty quantification capabilities in -`Tasmanian `_. +`Tasmanian `_. """ import numpy as np @@ -10,6 +10,11 @@ from libensemble.tools import parse_args from libensemble.tools.persistent_support import PersistentSupport +__all__ = [ + "sparse_grid_batched", + "sparse_grid_async", +] + def lex_le(x, y, tol=1e-12): """ @@ -195,7 +200,7 @@ def sparse_grid_batched(H, persis_info, gen_specs, libE_info): assert "sCriteria" in U grid.setSurplusRefinement(U["fTolerance"], U["iOutput"], U["sCriteria"]) - return H0, persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def sparse_grid_async(H, persis_info, gen_specs, libE_info): @@ -283,7 +288,7 @@ def get_refined_points(g, U): else: tag, Work, calc_in = ps.recv() - return [], persis_info, FINISHED_PERSISTENT_GEN_TAG + return None, persis_info, FINISHED_PERSISTENT_GEN_TAG def get_sparse_grid_specs(user_specs, sim_f, num_dims, num_outputs=1, mode="batched"): diff --git a/libensemble/gen_funcs/sampling.py b/libensemble/gen_funcs/sampling.py index 6cf0347fef..efe9eab407 100644 --- a/libensemble/gen_funcs/sampling.py +++ b/libensemble/gen_funcs/sampling.py @@ -1,7 +1,5 @@ """ -This module contains multiple generation functions for sampling a domain. All -use (and return) a random stream in ``persis_info``, given by the allocation -function. +This module contains multiple generator functions for sampling a domain. """ import numpy as np diff --git a/libensemble/gen_funcs/uniform_or_localopt.py b/libensemble/gen_funcs/uniform_or_localopt.py index 5e478cdcc1..fca0e70bb6 100644 --- a/libensemble/gen_funcs/uniform_or_localopt.py +++ b/libensemble/gen_funcs/uniform_or_localopt.py @@ -1,7 +1,7 @@ """ This module is a persistent generation function that performs a uniform random sample when ``libE_info["persistent"]`` isn't ``True``, or performs a -single persistent persistent nlopt local optimization run. +single persistent nlopt local optimization run. """ __all__ = ["uniform_or_localopt"] diff --git a/libensemble/manager.py b/libensemble/manager.py index 99321ce6a3..c0cf02500e 100644 --- a/libensemble/manager.py +++ b/libensemble/manager.py @@ -514,7 +514,7 @@ def _update_state_on_worker_msg(self, persis_info: dict, D_recv: dict, w: int) - self.live_data.live_update(self.hist) if D_recv.get("persis_info"): - persis_info[w].update(D_recv["persis_info"]) + persis_info.setdefault(int(w), {}).update(D_recv["persis_info"]) def _handle_msg_from_worker(self, persis_info: dict, w: int) -> None: """Handles a message from worker w""" @@ -580,7 +580,7 @@ def _final_receive_and_kill(self, persis_info: dict) -> (dict, int, int): rows_to_send = np.where(self.hist.H["sim_ended"] & ~self.hist.H["gen_informed"])[0] work = { "H_fields": self.gen_specs["persis_in"], - "persis_info": persis_info[w], + "persis_info": persis_info.get(w), "tag": PERSIS_STOP, "libE_info": {"persistent": True, "H_rows": rows_to_send}, } diff --git a/libensemble/resources/platforms.py b/libensemble/resources/platforms.py index 34f6cae1fc..cf4dde324a 100644 --- a/libensemble/resources/platforms.py +++ b/libensemble/resources/platforms.py @@ -144,7 +144,7 @@ class Aurora(Platform): # On SLURM systems, let srun assign free GPUs on the node -class Crusher(Platform): +class Frontier(Platform): mpi_runner: str = "srun" cores_per_node: int = 64 logical_cores_per_node: int = 128 @@ -154,19 +154,22 @@ class Crusher(Platform): scheduler_match_slots: bool = False -class Frontier(Platform): +# Example of a ROCM system +class GenericROCm(Platform): + mpi_runner: str = "mpich" + gpu_setting_type: str = "env" + gpu_setting_name: str = "ROCR_VISIBLE_DEVICES" + scheduler_match_slots: bool = True + + +class Lumi(Platform): mpi_runner: str = "srun" cores_per_node: int = 64 logical_cores_per_node: int = 128 - gpus_per_node: int = 8 - gpu_setting_type: str = "runner_default" - gpu_env_fallback: str = "ROCR_VISIBLE_DEVICES" - scheduler_match_slots: bool = False -# Example of a ROCM system -class GenericROCm(Platform): - mpi_runner: str = "mpich" +class LumiGPU(Lumi): + gpus_per_node: int = 8 gpu_setting_type: str = "env" gpu_setting_name: str = "ROCR_VISIBLE_DEVICES" scheduler_match_slots: bool = True @@ -202,16 +205,6 @@ class Polaris(Platform): scheduler_match_slots: bool = True -class Spock(Platform): - mpi_runner: str = "srun" - cores_per_node: int = 64 - logical_cores_per_node: int = 128 - gpus_per_node: int = 4 - gpu_setting_type: str = "runner_default" - gpu_setting_name: str = "ROCR_VISIBLE_DEVICES" - scheduler_match_slots: bool = False - - class Summit(Platform): mpi_runner: str = "jsrun" cores_per_node: int = 42 @@ -222,18 +215,6 @@ class Summit(Platform): scheduler_match_slots: bool = False -class Sunspot(Platform): - mpi_runner: str = "mpich" - runner_name: str = "mpiexec" - cores_per_node: int = 104 - logical_cores_per_node: int = 208 - gpus_per_node: int = 6 - tiles_per_gpu: int = 2 - gpu_setting_type: str = "env" - gpu_setting_name: str = "ZE_AFFINITY_MASK" - scheduler_match_slots: bool = True - - class Known_platforms(BaseModel): """A list of platforms with known configurations. @@ -274,24 +255,21 @@ class Known_platforms(BaseModel): aurora: Aurora = Aurora() generic_rocm: GenericROCm = GenericROCm() - crusher: Crusher = Crusher() frontier: Frontier = Frontier() + lumi: Lumi = Lumi() + lumi_g: LumiGPU = LumiGPU() perlmutter: Perlmutter = Perlmutter() perlmutter_c: PerlmutterCPU = PerlmutterCPU() perlmutter_g: PerlmutterGPU = PerlmutterGPU() polaris: Polaris = Polaris() - spock: Spock = Spock() summit: Summit = Summit() - sunspot: Sunspot = Sunspot() # Dictionary of known systems (or system partitions) detectable by domain name detect_systems = { - "crusher.olcf.ornl.gov": "crusher", "frontier.olcf.ornl.gov": "frontier", "hostmgmt.cm.aurora.alcf.anl.gov": "aurora", "hsn.cm.polaris.alcf.anl.gov": "polaris", - "spock.olcf.ornl.gov": "spock", "summit.olcf.ornl.gov": "summit", # Need to detect gpu count } @@ -309,6 +287,14 @@ def known_envs(): else: name = "perlmutter" logger.manager_warning("Perlmutter detected, but no compute partition detected. Are you on login nodes?") + if os.environ.get("SLURM_CLUSTER_NAME") == "lumi": + partition = os.environ.get("SLURM_JOB_PARTITION") + if not partition: + logger.manager_warning("LUMI detected, but no compute partition detected. Are you on login nodes?") + if partition and partition.endswith("-g"): + name = "lumi_g" + else: + name = "lumi" return name diff --git a/libensemble/sim_funcs/mock_sim.py b/libensemble/sim_funcs/mock_sim.py index 3099c9e89d..494c3dfa4c 100644 --- a/libensemble/sim_funcs/mock_sim.py +++ b/libensemble/sim_funcs/mock_sim.py @@ -2,10 +2,12 @@ def mock_sim(H, persis_info, sim_specs, libE_info): - """Places sim_specs["out"] from a numpy file into the outputs. Allows a user - to reproduce an existing run while, for example, capturing additional + """ + Places ``sim_specs["out"]`` from a numpy file into the outputs. Allows a + user to reproduce an existing run while, for example, capturing additional information from a gen. Requires a user to have set - sim_specs["user"]["history_file"] to point to a history file from a previous run. + ``sim_specs["user"]["history_file"]`` to point to a history file from a + previous run. """ hfile = sim_specs["user"]["history_file"] diff --git a/libensemble/sim_funcs/rosenbrock.py b/libensemble/sim_funcs/rosenbrock.py index 29da3e60b0..7925e0a2ba 100644 --- a/libensemble/sim_funcs/rosenbrock.py +++ b/libensemble/sim_funcs/rosenbrock.py @@ -1,10 +1,13 @@ -""" Sim function for Rosenbrock function. We include a @const input to the - gradient, and not the function evaluation, to scale down the gradient - so the Lipschitz and smoothness term is reduced. +""" +Sim function for Rosenbrock function. We include a @const input to the +gradient, and not the function evaluation, to scale down the gradient +so the Lipschitz and smoothness term is reduced. """ import numpy as np +__all__ = ["rosenbrock_eval"] + def EvaluateFunction(x, component=None): """ @@ -12,10 +15,15 @@ def EvaluateFunction(x, component=None): Parameters ---------- - x : np.ndarray - - input vector - component : int - - index + x : :class:`numpy.ndarray` + Input vector + component : int, optional + Index of the function component + + Returns + ------- + float + Evaluated function value """ assert len(x) % 2 == 0, "Length of input vector must be even" @@ -42,12 +50,17 @@ def EvaluateJacobian(x, component, const): Parameters ---------- - x : np.ndarray - - input vector + x : :class:`numpy.ndarray` + Input vector component : int - - index + Index of the function component const : float - - term to scale gradient by + Term to scale gradient by + + Returns + ------- + :class:`numpy.ndarray` + Jacobian of the function """ assert len(x) % 2 == 0, print("must be even lengthed input vector") @@ -71,6 +84,25 @@ def EvaluateJacobian(x, component, const): def rosenbrock_eval(H, persis_info, sim_specs, _): + """ + Evaluates the Rosenbrock function and its gradient + + Parameters + ---------- + H : dict + Dictionary containing input points + persis_info : dict + Persistent state information + sim_specs : dict + Simulation specifications + + Returns + ------- + tuple + - :class:`numpy.ndarray` - Evaluated function values + - dict - Updated `persis_info` + """ + if "params" in persis_info: const = persis_info["params"].get("const", 1000) else: diff --git a/libensemble/sim_funcs/var_resources.py b/libensemble/sim_funcs/var_resources.py index e3a9ad1175..65f91c18bd 100644 --- a/libensemble/sim_funcs/var_resources.py +++ b/libensemble/sim_funcs/var_resources.py @@ -80,7 +80,8 @@ def gpu_variable_resources(H, persis_info, sim_specs, libE_info): @input_fields(["x"]) @output_data([("f", float)]) def gpu_variable_resources_from_gen(H, persis_info, sim_specs, libE_info): - """Launches an app and assigns CPU and GPU resources as defined by the gen. + """ + Launches an app and assigns CPU and GPU resources as defined by the gen. Otherwise similar to gpu_variable_resources. """ diff --git a/libensemble/specs.py b/libensemble/specs.py index 494dd23f45..aa70018362 100644 --- a/libensemble/specs.py +++ b/libensemble/specs.py @@ -1,10 +1,10 @@ import random +import warnings from pathlib import Path from typing import Any, Callable, List, Optional, Tuple, Union import pydantic from pydantic import BaseModel, Field -import warnings from libensemble.alloc_funcs.give_sim_work_first import give_sim_work_first from libensemble.resources.platforms import Platform @@ -469,8 +469,8 @@ class LibeSpecs(BaseModel): dedicated_mode: Optional[bool] = False """ - Instructs libEnsemble to not run applications on resources where libEnsemble - processes (manager and workers) are running. + Instructs libEnsemble’s MPI executor not to run applications on nodes where + libEnsemble processes (manager and workers) are running. """ zero_resource_workers: Optional[List[int]] = [] diff --git a/libensemble/tests/functionality_tests/test_1d_sampling_from_files.py b/libensemble/tests/functionality_tests/test_1d_sampling_from_files.py index 70110ec828..d80025d238 100644 --- a/libensemble/tests/functionality_tests/test_1d_sampling_from_files.py +++ b/libensemble/tests/functionality_tests/test_1d_sampling_from_files.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_1d_sampling_from_yaml.py - python test_1d_sampling_from_yaml.py --nworkers 3 --comms local + python test_1d_sampling_from_yaml.py --nworkers 3 python test_1d_sampling_from_yaml.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py b/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py index 4033a4cfad..645228d137 100644 --- a/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py +++ b/libensemble/tests/functionality_tests/test_1d_sampling_with_profile.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_1d_sampling_with_profile.py - python test_1d_sampling_with_profile.py --nworkers 3 --comms local + python test_1d_sampling_with_profile.py --nworkers 3 python test_1d_sampling_with_profile.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_1d_splitcomm.py b/libensemble/tests/functionality_tests/test_1d_splitcomm.py index 528a456940..de73660d70 100644 --- a/libensemble/tests/functionality_tests/test_1d_splitcomm.py +++ b/libensemble/tests/functionality_tests/test_1d_splitcomm.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_1d_sampling.py - python test_1d_sampling.py --nworkers 3 --comms local + python test_1d_sampling.py --nworkers 3 python test_1d_sampling.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_1d_subcomm.py b/libensemble/tests/functionality_tests/test_1d_subcomm.py index 5dd49d9d3c..7f607c31c9 100644 --- a/libensemble/tests/functionality_tests/test_1d_subcomm.py +++ b/libensemble/tests/functionality_tests/test_1d_subcomm.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_1d_sampling.py - python test_1d_sampling.py --nworkers 3 --comms local + python test_1d_sampling.py --nworkers 3 python test_1d_sampling.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_1d_super_simple.py b/libensemble/tests/functionality_tests/test_1d_super_simple.py index 0bbc4a56c0..e84255714f 100644 --- a/libensemble/tests/functionality_tests/test_1d_super_simple.py +++ b/libensemble/tests/functionality_tests/test_1d_super_simple.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_1d_sampling.py - python test_1d_sampling.py --nworkers 3 --comms local + python test_1d_sampling.py --nworkers 3 python test_1d_sampling.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_GPU_gen_resources.py b/libensemble/tests/functionality_tests/test_GPU_gen_resources.py index 0fc8192f7c..d77088d7e4 100644 --- a/libensemble/tests/functionality_tests/test_GPU_gen_resources.py +++ b/libensemble/tests/functionality_tests/test_GPU_gen_resources.py @@ -17,7 +17,7 @@ Execute via one of the following commands (e.g., 4 workers): mpiexec -np 5 python test_GPU_gen_resources.py - python test_GPU_gen_resources.py --comms local --nworkers 4 + python test_GPU_gen_resources.py --nworkers 4 When running with the above command, the number of concurrent evaluations of the objective function will be 4, as one of the five workers will be the diff --git a/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py b/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py index d896d3d848..7d99968629 100644 --- a/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py +++ b/libensemble/tests/functionality_tests/test_active_persistent_worker_abort.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_6-hump_camel_active_persistent_worker_abort.py - python test_6-hump_camel_active_persistent_worker_abort.py --nworkers 3 --comms local + python test_6-hump_camel_active_persistent_worker_abort.py --nworkers 3 python test_6-hump_camel_active_persistent_worker_abort.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_calc_exception.py b/libensemble/tests/functionality_tests/test_calc_exception.py index 6321bea138..361e1be3a8 100644 --- a/libensemble/tests/functionality_tests/test_calc_exception.py +++ b/libensemble/tests/functionality_tests/test_calc_exception.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_calc_exception.py - python test_calc_exception.py --nworkers 3 --comms local + python test_calc_exception.py --nworkers 3 python test_calc_exception.py --nworkers 3 --comms tcp """ diff --git a/libensemble/tests/functionality_tests/test_cancel_in_alloc.py b/libensemble/tests/functionality_tests/test_cancel_in_alloc.py index de7c25d303..d2c005a040 100644 --- a/libensemble/tests/functionality_tests/test_cancel_in_alloc.py +++ b/libensemble/tests/functionality_tests/test_cancel_in_alloc.py @@ -7,7 +7,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_cancel_in_alloc.py - python test_cancel_in_alloc.py --nworkers 3 --comms local + python test_cancel_in_alloc.py --nworkers 3 python test_cancel_in_alloc.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_comms.py b/libensemble/tests/functionality_tests/test_comms.py index 2e48eb8743..52c3e0771a 100644 --- a/libensemble/tests/functionality_tests/test_comms.py +++ b/libensemble/tests/functionality_tests/test_comms.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_comms.py - python test_comms.py --nworkers 3 --comms local + python test_comms.py --nworkers 3 python test_comms.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be N-1. diff --git a/libensemble/tests/functionality_tests/test_elapsed_time_abort.py b/libensemble/tests/functionality_tests/test_elapsed_time_abort.py index 7c7d6ec823..9e7ab97049 100644 --- a/libensemble/tests/functionality_tests/test_elapsed_time_abort.py +++ b/libensemble/tests/functionality_tests/test_elapsed_time_abort.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_6-hump_camel_elapsed_time_abort.py - python test_6-hump_camel_elapsed_time_abort.py --nworkers 3 --comms local + python test_6-hump_camel_elapsed_time_abort.py --nworkers 3 python test_6-hump_camel_elapsed_time_abort.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py b/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py index 2b601efe33..fe3d8dad8e 100644 --- a/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py +++ b/libensemble/tests/functionality_tests/test_evaluate_existing_plus_gen.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_evaluate_existing_sample.py - python test_evaluate_existing_sample.py --nworkers 3 --comms local + python test_evaluate_existing_sample.py --nworkers 3 python test_evaluate_existing_sample.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py b/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py index fd96275a20..1b61bf8d25 100644 --- a/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py +++ b/libensemble/tests/functionality_tests/test_executor_hworld_pass_fail.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_executor_hworld.py - python test_executor_hworld.py --nworkers 3 --comms local + python test_executor_hworld.py --nworkers 3 python test_executor_hworld.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py b/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py index ec532fe6f0..496eced316 100644 --- a/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py +++ b/libensemble/tests/functionality_tests/test_executor_hworld_timeout.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_executor_hworld.py - python test_executor_hworld.py --nworkers 3 --comms local + python test_executor_hworld.py --nworkers 3 python test_executor_hworld.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_executor_simple.py b/libensemble/tests/functionality_tests/test_executor_simple.py index 64e984dba6..ac4c201a7a 100644 --- a/libensemble/tests/functionality_tests/test_executor_simple.py +++ b/libensemble/tests/functionality_tests/test_executor_simple.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_executor_hworld.py - python test_executor_hworld.py --nworkers 3 --comms local + python test_executor_hworld.py --nworkers 3 python test_executor_hworld.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_fast_alloc.py b/libensemble/tests/functionality_tests/test_fast_alloc.py index 14fc0ecb92..a2b6a75666 100644 --- a/libensemble/tests/functionality_tests/test_fast_alloc.py +++ b/libensemble/tests/functionality_tests/test_fast_alloc.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_fast_alloc.py - python test_fast_alloc.py --nworkers 3 --comms local + python test_fast_alloc.py --nworkers 3 The number of concurrent evaluations of the objective function will be 4-1=3. """ diff --git a/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py b/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py index bb23e4a78b..31e537a31d 100644 --- a/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py +++ b/libensemble/tests/functionality_tests/test_mpi_gpu_settings.py @@ -30,7 +30,7 @@ Execute via one of the following commands (e.g. 5 workers): mpiexec -np 6 python test_mpi_gpu_settings.py - python test_mpi_gpu_settings.py --comms local --nworkers 5 + python test_mpi_gpu_settings.py --nworkers 5 When running with the above command, the number of concurrent evaluations of the objective function will be 4, as one of the five workers will be the @@ -203,7 +203,7 @@ del libE_specs["platform_specs"] # Fourth set - use platform setting ------------------------------------------------------------ - for platform in ["summit", "crusher", "perlmutter_g", "polaris", "aurora"]: + for platform in ["summit", "frontier", "perlmutter_g", "polaris", "aurora"]: print(f"\nRunning GPU setting checks (via known platform) for {platform} ------------------- ") libE_specs["platform"] = platform @@ -221,7 +221,7 @@ del libE_specs["platform"] # Fifth set - use platform environment setting ----------------------------------------------- - for platform in ["summit", "crusher", "perlmutter_g", "polaris", "aurora"]: + for platform in ["summit", "frontier", "perlmutter_g", "polaris", "aurora"]: print(f"\nRunning GPU setting checks (via known platform env. variable) for {platform} ----- ") os.environ["LIBE_PLATFORM"] = platform diff --git a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py index 2265f71758..814f5086cb 100644 --- a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py +++ b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_env.py @@ -10,7 +10,7 @@ Execute via one of the following commands (e.g., 5 workers): mpiexec -np 6 python test_mpi_gpu_settings_env.py - python test_mpi_gpu_settings_env.py --comms local --nworkers 5 + python test_mpi_gpu_settings_env.py --nworkers 5 When running with the above command, the number of concurrent evaluations of the objective function will be 4, as one of the five workers will be the diff --git a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py index f299020f3f..3c884c7dd2 100644 --- a/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py +++ b/libensemble/tests/functionality_tests/test_mpi_gpu_settings_mock_nodes_multi_task.py @@ -11,7 +11,7 @@ Execute via one of the following commands (e.g. 5 workers): mpiexec -np 6 python test_mpi_gpu_settings_mock_nodes_multi_task.py - python test_mpi_gpu_settings_mock_nodes_multi_task.py --comms local --nworkers 5 + python test_mpi_gpu_settings_mock_nodes_multi_task.py --nworkers 5 When running with the above command, the number of concurrent evaluations of the objective function will be 4, as one of the five workers will be the diff --git a/libensemble/tests/functionality_tests/test_mpi_runners.py b/libensemble/tests/functionality_tests/test_mpi_runners.py index 12cc658054..af00471d59 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_mpi_runners.py - python test_mpi_runners.py --nworkers 3 --comms local + python test_mpi_runners.py --nworkers 3 python test_mpi_runners.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py b/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py index a3d16c2722..7266678ff2 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_subnode.py @@ -5,7 +5,7 @@ Execute via one of the following commands (e.g. 4 workers): mpiexec -np 5 python test_mpi_runners_subnode.py - python test_mpi_runners_subnode.py --nworkers 4 --comms local + python test_mpi_runners_subnode.py --nworkers 4 python test_mpi_runners_subnode.py --nworkers 4 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py index 5ae749c7d4..a5145965b9 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_subnode_uneven.py @@ -5,7 +5,7 @@ Execute via one of the following commands (e.g. 5 workers): mpiexec -np 6 python test_mpi_runners_subnode_uneven.py - python test_mpi_runners_subnode_uneven.py --nworkers 5 --comms local + python test_mpi_runners_subnode_uneven.py --nworkers 5 python test_mpi_runners_subnode_uneven.py --nworkers 5 --comms tcp """ diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py index 45c8f2c493..77975e200d 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_supernode_uneven.py @@ -5,7 +5,7 @@ Execute via one of the following commands (e.g. 5 workers): mpiexec -np 6 python test_mpi_runners_supernode_uneven.py - python test_mpi_runners_supernode_uneven.py --nworkers 5 --comms local + python test_mpi_runners_supernode_uneven.py --nworkers 5 """ import numpy as np diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py index 1e189ab47d..cc73d0e427 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_subnode_uneven.py @@ -5,7 +5,7 @@ Execute via one of the following commands (e.g. 6 workers - one is zero resource): mpiexec -np 7 python test_mpi_runners_zrw_subnode_uneven.py - python test_mpi_runners_zrw_subnode_uneven.py --nworkers 6 --comms local + python test_mpi_runners_zrw_subnode_uneven.py --nworkers 6 python test_mpi_runners_zrw_subnode_uneven.py --nworkers 6 --comms tcp The resource sets are split unevenly between the two nodes (e.g. 3 and 2). diff --git a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py index 9b2521ec06..640d613bff 100644 --- a/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py +++ b/libensemble/tests/functionality_tests/test_mpi_runners_zrw_supernode_uneven.py @@ -5,7 +5,7 @@ Execute via one of the following commands (e.g. 6 workers - one is zero resource): mpiexec -np 7 python test_mpi_runners_zrw_supernode_uneven.py - python test_mpi_runners_zrw_supernode_uneven.py --nworkers 6 --comms local + python test_mpi_runners_zrw_supernode_uneven.py --nworkers 6 """ import numpy as np diff --git a/libensemble/tests/functionality_tests/test_new_field.py b/libensemble/tests/functionality_tests/test_new_field.py index 2d1f0e3a84..c130b1d6e3 100644 --- a/libensemble/tests/functionality_tests/test_new_field.py +++ b/libensemble/tests/functionality_tests/test_new_field.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_1d_sampling.py - python test_1d_sampling.py --nworkers 3 --comms local + python test_1d_sampling.py --nworkers 3 python test_1d_sampling.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py b/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py index 138d3b9e5c..f1f0616663 100644 --- a/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py +++ b/libensemble/tests/functionality_tests/test_persistent_sim_uniform_sampling.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_sim_uniform_sampling.py - python test_persistent_sim_uniform_sampling.py --nworkers 3 --comms local + python test_persistent_sim_uniform_sampling.py --nworkers 3 python test_persistent_sim_uniform_sampling.py --nworkers 3 --comms tcp When running with the above command, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py b/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py index c61753292c..68c8aaaa05 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 5 python test_persistent_uniform_gen_decides_stop.py - python test_persistent_uniform_gen_decides_stop.py --nworkers 4 --comms local + python test_persistent_uniform_gen_decides_stop.py --nworkers 4 python test_persistent_uniform_gen_decides_stop.py --nworkers 4 --comms tcp The number of concurrent evaluations of the objective function with 2 gens will be 2: diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py index 305521a02b..81a18a5285 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_uniform_sampling.py - python test_persistent_uniform_sampling.py --nworkers 3 --comms local + python test_persistent_uniform_sampling.py --nworkers 3 python test_persistent_uniform_sampling.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py index 2a68a2061b..77d9a6a7b8 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_async.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_uniform_sampling_async.py - python test_persistent_uniform_sampling_async.py --nworkers 3 --comms local + python test_persistent_uniform_sampling_async.py --nworkers 3 python test_persistent_uniform_sampling_async.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py index b6bcc033aa..15c2d5b097 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_cancel.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_uniform_sampling_cancel.py - python test_persistent_uniform_sampling_cancel.py --nworkers 3 --comms local + python test_persistent_uniform_sampling_cancel.py --nworkers 3 python test_persistent_uniform_sampling_cancel.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py index 4fe3928998..5425578849 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_nonblocking.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_uniform_sampling_nonblocking.py - python test_persistent_uniform_sampling_nonblocking.py --nworkers 3 --comms local + python test_persistent_uniform_sampling_nonblocking.py --nworkers 3 python test_persistent_uniform_sampling_nonblocking.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py index 68decd00f6..9cc92dadd3 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_sampling_running_mean.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g., 3 workers): mpiexec -np 4 python test_persistent_uniform_sampling_adv.py - python test_persistent_uniform_sampling_running_mean.py --nworkers 3 --comms local + python test_persistent_uniform_sampling_running_mean.py --nworkers 3 python test_persistent_uniform_sampling_running_mean.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py b/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py index 3f8380e6ba..b4c30f9d2d 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_per_calc.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_sim_dirs_per_calc.py - python test_sim_dirs_per_calc.py --nworkers 3 --comms local + python test_sim_dirs_per_calc.py --nworkers 3 python test_sim_dirs_per_calc.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py b/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py index d8060d8a4a..69bb34ab84 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_per_worker.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_sim_dirs_per_worker.py - python test_sim_dirs_per_worker.py --nworkers 3 --comms local + python test_sim_dirs_per_worker.py --nworkers 3 python test_sim_dirs_per_worker.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py b/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py index 4487725316..229f6d5f54 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_with_exception.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_sim_dirs_with_exception.py - python test_sim_dirs_with_exception.py --nworkers 3 --comms local + python test_sim_dirs_with_exception.py --nworkers 3 python test_sim_dirs_with_exception.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py b/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py index 46e3f6266c..c73eeabadf 100644 --- a/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py +++ b/libensemble/tests/functionality_tests/test_sim_dirs_with_gen_dirs.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_sim_dirs_with_gen_dirs.py - python test_sim_dirs_with_gen_dirs.py --nworkers 3 --comms local + python test_sim_dirs_with_gen_dirs.py --nworkers 3 python test_sim_dirs_with_gen_dirs.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_sim_input_dir_option.py b/libensemble/tests/functionality_tests/test_sim_input_dir_option.py index 2621499058..7dc4da7057 100644 --- a/libensemble/tests/functionality_tests/test_sim_input_dir_option.py +++ b/libensemble/tests/functionality_tests/test_sim_input_dir_option.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_sim_input_dir_option.py - python test_sim_input_dir_option.py --nworkers 3 --comms local + python test_sim_input_dir_option.py --nworkers 3 python test_sim_input_dir_option.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_stats_output.py b/libensemble/tests/functionality_tests/test_stats_output.py index 921f55f9c2..58d012cdeb 100644 --- a/libensemble/tests/functionality_tests/test_stats_output.py +++ b/libensemble/tests/functionality_tests/test_stats_output.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_stats_output.py - python test_stats_output.py --nworkers 3 --comms local + python test_stats_output.py --nworkers 3 The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling.py b/libensemble/tests/functionality_tests/test_uniform_sampling.py index 9af49f0b9a..2867f94df9 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_uniform_sampling.py - python test_uniform_sampling.py --nworkers 3 --comms local + python test_uniform_sampling.py --nworkers 3 python test_uniform_sampling.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py b/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py index 17675a7187..6080fc3ce3 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_cancel.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_uniform_sampling_cancel.py - python test_uniform_sampling_cancel.py --nworkers 3 --comms local + python test_uniform_sampling_cancel.py --nworkers 3 python test_uniform_sampling_cancel.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py b/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py index 14f9b798d6..ec126607ef 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_then_persistent_localopt_runs.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_uniform_sampling_then_persistent_localopt_runs.py - python test_uniform_sampling_then_persistent_localopt_runs.py --nworkers 3 --comms local + python test_uniform_sampling_then_persistent_localopt_runs.py --nworkers 3 python test_uniform_sampling_then_persistent_localopt_runs.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py b/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py index 049f886a7a..9123f7db17 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_uniform_sampling_with_variable_resources.py - python test_uniform_sampling_with_variable_resources.py --nworkers 3 --comms local + python test_uniform_sampling_with_variable_resources.py --nworkers 3 The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_worker_exceptions.py b/libensemble/tests/functionality_tests/test_worker_exceptions.py index 0e87eb372c..435e621e16 100644 --- a/libensemble/tests/functionality_tests/test_worker_exceptions.py +++ b/libensemble/tests/functionality_tests/test_worker_exceptions.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_worker_exceptions.py - python test_worker_exceptions.py --nworkers 3 --comms local + python test_worker_exceptions.py --nworkers 3 python test_worker_exceptions.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_workflow_dir.py b/libensemble/tests/functionality_tests/test_workflow_dir.py index fd76f8b68f..1abbc10bcf 100644 --- a/libensemble/tests/functionality_tests/test_workflow_dir.py +++ b/libensemble/tests/functionality_tests/test_workflow_dir.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_sim_input_dir_option.py - python test_sim_input_dir_option.py --nworkers 3 --comms local + python test_sim_input_dir_option.py --nworkers 3 python test_sim_input_dir_option.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_zero_resource_workers.py b/libensemble/tests/functionality_tests/test_zero_resource_workers.py index 21ff25f10f..c8f0786d06 100644 --- a/libensemble/tests/functionality_tests/test_zero_resource_workers.py +++ b/libensemble/tests/functionality_tests/test_zero_resource_workers.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_zero_resource_workers.py - python test_zero_resource_workers.py --nworkers 3 --comms local + python test_zero_resource_workers.py --nworkers 3 python test_zero_resource_workers.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py b/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py index 5ef6e6ff74..69ea2b559c 100644 --- a/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py +++ b/libensemble/tests/functionality_tests/test_zero_resource_workers_subnode.py @@ -7,7 +7,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_zero_resource_workers_subnode.py - python test_zero_resource_workers_subnode.py --nworkers 3 --comms local + python test_zero_resource_workers_subnode.py --nworkers 3 python test_zero_resource_workers_subnode.py --nworkers 3 --comms tcp """ diff --git a/libensemble/tests/regression_tests/test_1d_sampling.py b/libensemble/tests/regression_tests/test_1d_sampling.py index b7e196659f..edecabb668 100644 --- a/libensemble/tests/regression_tests/test_1d_sampling.py +++ b/libensemble/tests/regression_tests/test_1d_sampling.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_1d_sampling.py - python test_1d_sampling.py --nworkers 3 --comms local + python test_1d_sampling.py --nworkers 3 python test_1d_sampling.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/regression_tests/test_2d_sampling.py b/libensemble/tests/regression_tests/test_2d_sampling.py index 5e888f51a7..8164c2844a 100644 --- a/libensemble/tests/regression_tests/test_2d_sampling.py +++ b/libensemble/tests/regression_tests/test_2d_sampling.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_2d_sampling.py - python test_2d_sampling.py --nworkers 3 --comms local + python test_2d_sampling.py --nworkers 3 python test_2d_sampling.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/regression_tests/test_GPU_variable_resources.py b/libensemble/tests/regression_tests/test_GPU_variable_resources.py index a44a9d24d0..b6b3197f90 100644 --- a/libensemble/tests/regression_tests/test_GPU_variable_resources.py +++ b/libensemble/tests/regression_tests/test_GPU_variable_resources.py @@ -14,7 +14,7 @@ Execute via one of the following commands (e.g. 5 workers): mpiexec -np 6 python test_GPU_variable_resources.py - python test_GPU_variable_resources.py --comms local --nworkers 5 + python test_GPU_variable_resources.py --nworkers 5 When running with the above command, the number of concurrent evaluations of the objective function will be 4, as one of the five workers will be the diff --git a/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py b/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py index 261945963f..2b583d4f06 100644 --- a/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py +++ b/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py @@ -18,7 +18,7 @@ Execute via one of the following commands (e.g. 9 workers): mpiexec -np 10 python test_GPU_variable_resources_multi_task.py - python test_GPU_variable_resources_multi_task.py --comms local --nworkers 9 + python test_GPU_variable_resources_multi_task.py --nworkers 9 When running with the above command, the number of concurrent evaluations of the objective function will be 8, as one of the nine workers will be the diff --git a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py index 480dd53ec5..8f1ee674ce 100644 --- a/libensemble/tests/regression_tests/test_evaluate_existing_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_existing_sample.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_evaluate_existing_sample.py - python test_evaluate_existing_sample.py --nworkers 3 --comms local + python test_evaluate_existing_sample.py --nworkers 3 python test_evaluate_existing_sample.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py index 1574e8d57b..481db84191 100644 --- a/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py +++ b/libensemble/tests/regression_tests/test_evaluate_mixed_sample.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_evaluate_mixed_sample.py - python test_evaluate_mixed_sample.py --nworkers 3 --comms local + python test_evaluate_mixed_sample.py --nworkers 3 python test_evaluate_mixed_sample.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/regression_tests/test_gpCAM.py b/libensemble/tests/regression_tests/test_gpCAM.py index e1bc1e4049..218ecfc918 100644 --- a/libensemble/tests/regression_tests/test_gpCAM.py +++ b/libensemble/tests/regression_tests/test_gpCAM.py @@ -3,23 +3,25 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_gpCAM.py - python test_gpCAM.py --nworkers 3 --comms local + python test_gpCAM.py --nworkers 3 When running with the above commands, the number of concurrent evaluations of the objective function will be 2, as one of the three workers will be the persistent generator. -See libensemble.gen_funcs.persistent_gpCAM for more details about the generator -setup. +Runs three variants of gpCAM. The first two use the posterior covariance +sampling method, whereby the second run uses the grid approach and uses +the points from the first run as it’s test points. The third run uses the +gpCAM ask/tell interface. -Requires numpy<2. +See libensemble.gen_funcs.persistent_gpCAM for more details about the +generator setup. """ # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: mpi local # TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true -# TESTSUITE_EXCLUDE: true import sys import warnings @@ -27,18 +29,17 @@ import numpy as np from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f -from libensemble.gen_funcs.persistent_gpCAM import persistent_gpCAM_ask_tell, persistent_gpCAM_simple +from libensemble.gen_funcs.persistent_gpCAM import persistent_gpCAM, persistent_gpCAM_covar # Import libEnsemble items for this test from libensemble.libE import libE from libensemble.sim_funcs.rosenbrock import rosenbrock_eval as sim_f from libensemble.tools import add_unique_random_streams, parse_args, save_libE_output -# Main block is necessary only when using local comms with spawn start method (default on macOS and Windows). - warnings.filterwarnings("ignore", message="Default hyperparameter_bounds") +warnings.filterwarnings("ignore", message="Hyperparameters initialized") - +# Main block is necessary only when using local comms with spawn start method (default on macOS and Windows). if __name__ == "__main__": nworkers, is_manager, libE_specs, _ = parse_args() @@ -70,7 +71,7 @@ for inst in range(3): if inst == 0: - gen_specs["gen_f"] = persistent_gpCAM_simple + gen_specs["gen_f"] = persistent_gpCAM_covar num_batches = 10 exit_criteria = {"sim_max": num_batches * batch_size, "wallclock_max": 300} libE_specs["save_every_k_gens"] = 150 @@ -82,7 +83,7 @@ del libE_specs["H_file_prefix"] del libE_specs["save_every_k_gens"] elif inst == 2: - gen_specs["gen_f"] = persistent_gpCAM_ask_tell + gen_specs["gen_f"] = persistent_gpCAM num_batches = 3 # Few because the ask_tell gen can be slow gen_specs["user"]["ask_max_iter"] = 1 # For quicker test exit_criteria = {"sim_max": num_batches * batch_size, "wallclock_max": 300} diff --git a/libensemble/tests/regression_tests/test_inverse_bayes_example.py b/libensemble/tests/regression_tests/test_inverse_bayes_example.py index 993fdfe12c..f1e5d1cc3a 100644 --- a/libensemble/tests/regression_tests/test_inverse_bayes_example.py +++ b/libensemble/tests/regression_tests/test_inverse_bayes_example.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_inverse_bayes_example.py - python test_inverse_bayes_example.py --nworkers 3 --comms local + python test_inverse_bayes_example.py --nworkers 3 python test_inverse_bayes_example.py --nworkers 3 --comms tcp Debugging: diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py index 14620c7e07..6e19930691 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_dfols.py @@ -5,7 +5,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_dfols.py - python test_persistent_aposmm_dfols.py --nworkers 3 --comms local + python test_persistent_aposmm_dfols.py --nworkers 3 python test_persistent_aposmm_dfols.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py b/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py index c5500d3ac9..b197dc3f07 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_exception.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_exception.py - python test_persistent_aposmm_exception.py --nworkers 3 --comms local + python test_persistent_aposmm_exception.py --nworkers 3 When running with the above commands, the number of concurrent evaluations of the objective function will be 2, as one of the three workers will be the diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py b/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py index dd0a86b5ba..7523704a0b 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py @@ -14,9 +14,9 @@ These values are then mapped to the normalized emittance - . Execute via one of the following commands: - mpiexec -np 3 python test_persistent_aposmm_ibcdfo.py - python test_persistent_aposmm_ibcdfo.py --nworkers 2 --comms local -Both will run with 1 manager, 1 worker running APOSMM+IBCDFO), and 1 worker + mpiexec -np 3 python test_persistent_aposmm_ibcdfo_pounders.py + python test_persistent_aposmm_ibcdfo_pounders.py --nworkers 2 +Both will run with 1 manager, 1 worker running APOSMM+IBCDFO, and 1 worker doing the simulation evaluations. """ diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py b/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py index 681133016e..3cf69bf5dd 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_nlopt.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_nlopt.py - python test_persistent_aposmm_nlopt.py --nworkers 3 --comms local + python test_persistent_aposmm_nlopt.py --nworkers 3 python test_persistent_aposmm_nlopt.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_periodic.py b/libensemble/tests/regression_tests/test_persistent_aposmm_periodic.py index 150bc73a1d..d99e8802a0 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_periodic.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_periodic.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_periodic.py - python test_persistent_aposmm_periodic.py --nworkers 3 --comms local + python test_persistent_aposmm_periodic.py --nworkers 3 When running with the above commands, the number of concurrent evaluations of the objective function will be 2, as one of the three workers will be the diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py b/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py index d6787d570c..0a8bfb382c 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_pounders.py @@ -5,7 +5,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_pounders.py - python test_persistent_aposmm_pounders.py --nworkers 3 --comms local + python test_persistent_aposmm_pounders.py --nworkers 3 python test_persistent_aposmm_pounders.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py index e0c393bc9f..ee4ec225b3 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_scipy.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_scipy.py - python test_persistent_aposmm_scipy.py --nworkers 3 --comms local + python test_persistent_aposmm_scipy.py --nworkers 3 python test_persistent_aposmm_scipy.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py index 30fa297120..4d77df0ee6 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_blmvm.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_tao_blmvm.py - python test_persistent_aposmm_tao_blmvm.py --nworkers 3 --comms local + python test_persistent_aposmm_tao_blmvm.py --nworkers 3 python test_persistent_aposmm_tao_blmvm.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py index 0339e25326..e8401217e0 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_tao_nm.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_tao_nm.py - python test_persistent_aposmm_tao_nm.py --nworkers 3 --comms local + python test_persistent_aposmm_tao_nm.py --nworkers 3 python test_persistent_aposmm_tao_nm.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py b/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py index e547beaaff..e61843fd71 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_timeout.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_timeout.py - python test_persistent_aposmm_timeout.py --nworkers 3 --comms local + python test_persistent_aposmm_timeout.py --nworkers 3 python test_persistent_aposmm_timeout.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py index 23e58b134a..f2d2f09cc0 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_with_grad.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_aposmm_with_grad.py - python test_persistent_aposmm_with_grad.py --nworkers 3 --comms local + python test_persistent_aposmm_with_grad.py --nworkers 3 python test_persistent_aposmm_with_grad.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py b/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py index 35acc541e3..8c589161ad 100644 --- a/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py +++ b/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py @@ -2,24 +2,24 @@ Example of multi-fidelity optimization using a persistent GP gen_func (calling Ax). -Execute via one of the following commands (e.g. 5 workers): - mpiexec -np 5 python test_persistent_gp_multitask_ax.py - python test_persistent_gp_multitask_ax.py --nworkers 4 --comms local - python test_persistent_gp_multitask_ax.py --nworkers 4 --comms tcp +This test uses the gen_on_manager option (persistent generator runs on +a thread). Therefore nworkers is the number of simulation workers. + +Execute via one of the following commands: + mpiexec -np 4 python test_persistent_gp_multitask_ax.py + python test_persistent_gp_multitask_ax.py --nworkers 3 + python test_persistent_gp_multitask_ax.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of -the objective function will be 3, as one of the three workers will be the -persistent generator. +the objective function will be 3. -Requires numpy<2. """ # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: local mpi -# TESTSUITE_NPROCS: 5 +# TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true # TESTSUITE_OS_SKIP: OSX -# TESTSUITE_EXCLUDE: true import warnings @@ -50,6 +50,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info): z = 8 elif task == "cheap_model": z = 1 + print('in sim', task) libE_output = np.zeros(1, dtype=sim_specs["out"]) calc_status = WORKER_DONE @@ -63,6 +64,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info): # Main block is necessary only when using local comms with spawn start method (default on macOS and Windows). if __name__ == "__main__": nworkers, is_manager, libE_specs, _ = parse_args() + libE_specs["gen_on_manager"] = True mt_params = { "name_hifi": "expensive_model", diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py index de8f15597e..39cf11b5de 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_surmise_calib.py - python test_persistent_surmise_calib.py --nworkers 3 --comms local + python test_persistent_surmise_calib.py --nworkers 3 python test_persistent_surmise_calib.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py index 56698bc680..11095f61f2 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_surmise_killsims.py - python test_persistent_surmise_killsims.py --nworkers 3 --comms local + python test_persistent_surmise_killsims.py --nworkers 3 python test_persistent_surmise_killsims.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of @@ -74,7 +74,7 @@ # Subprocess variant creates input and output files for each sim libE_specs["sim_dirs_make"] = True # To keep all - make sim dirs - # libE_specs["use_worker_dirs"] = True # To overwrite - make worker dirs only + libE_specs["kill_canceled_sims"] = True # Rename ensemble dir for non-interference with other regression tests en_suffix = str(nworkers) + "_" + libE_specs.get("comms") diff --git a/libensemble/tests/regression_tests/test_persistent_tasmanian.py b/libensemble/tests/regression_tests/test_persistent_tasmanian.py index 8083a80df4..269c4ba595 100644 --- a/libensemble/tests/regression_tests/test_persistent_tasmanian.py +++ b/libensemble/tests/regression_tests/test_persistent_tasmanian.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_tasmanian.py - python test_persistent_tasmanian.py --nworkers 3 --comms local + python test_persistent_tasmanian.py --nworkers 3 python test_persistent_tasmanian.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_persistent_tasmanian_async.py b/libensemble/tests/regression_tests/test_persistent_tasmanian_async.py index 6eabed2f6e..f21544d185 100644 --- a/libensemble/tests/regression_tests/test_persistent_tasmanian_async.py +++ b/libensemble/tests/regression_tests/test_persistent_tasmanian_async.py @@ -3,7 +3,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_persistent_tasmanian_async.py - python test_persistent_tasmanian_async.py --nworkers 3 --comms local + python test_persistent_tasmanian_async.py --nworkers 3 python test_persistent_tasmanian_async.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/regression_tests/test_proxystore_integration.py b/libensemble/tests/regression_tests/test_proxystore_integration.py index a900f876d0..885e6fe6bf 100644 --- a/libensemble/tests/regression_tests/test_proxystore_integration.py +++ b/libensemble/tests/regression_tests/test_proxystore_integration.py @@ -4,7 +4,7 @@ Execute via one of the following commands (e.g. 3 workers): mpiexec -np 4 python test_evaluate_existing_sample.py - python test_evaluate_existing_sample.py --nworkers 3 --comms local + python test_evaluate_existing_sample.py --nworkers 3 python test_evaluate_existing_sample.py --nworkers 3 --comms tcp The number of concurrent evaluations of the objective function will be 4-1=3. diff --git a/libensemble/tests/regression_tests/test_with_app_persistent_aposmm_tao_nm.py b/libensemble/tests/regression_tests/test_with_app_persistent_aposmm_tao_nm.py index a314e6bafc..6d21d50d6f 100644 --- a/libensemble/tests/regression_tests/test_with_app_persistent_aposmm_tao_nm.py +++ b/libensemble/tests/regression_tests/test_with_app_persistent_aposmm_tao_nm.py @@ -7,7 +7,7 @@ Execute via one of the following commands (e.g., 3 workers): mpiexec -np 4 python test_persistent_aposmm_tao_nm.py - python test_with_app_persistent_aposmm_tao_nm.py --nworkers 3 --comms local + python test_with_app_persistent_aposmm_tao_nm.py --nworkers 3 python test_with_app_persistent_aposmm_tao_nm.py --nworkers 3 --comms tcp When running with the above commands, the number of concurrent evaluations of diff --git a/libensemble/tests/run_tests.py b/libensemble/tests/run_tests.py index e8f7de57eb..073ab1a541 100755 --- a/libensemble/tests/run_tests.py +++ b/libensemble/tests/run_tests.py @@ -360,7 +360,8 @@ def run_regression_tests(root_dir, python_exec, args, current_os): user_comms_list = ["mpi", "local", "tcp"] print_heading(f"Running regression tests (comms: {', '.join(user_comms_list)})") - build_forces(root_dir) # Build forces.x before running tests + if not REG_LIST_TESTS_ONLY: + build_forces(root_dir) # Build forces.x before running tests reg_test_list = REG_TEST_LIST reg_test_files = [] diff --git a/libensemble/tests/scaling_tests/forces/forces_adv/readme.md b/libensemble/tests/scaling_tests/forces/forces_adv/readme.md index e30f10d2d3..2d0daf9f79 100644 --- a/libensemble/tests/scaling_tests/forces/forces_adv/readme.md +++ b/libensemble/tests/scaling_tests/forces/forces_adv/readme.md @@ -64,10 +64,6 @@ you will need to replace the following templated values if on a Cobalt system: The included scripts are. -* summit_submit_mproc.sh: - - Example Summit submission script to run libEnsemble in central mode on the Summit launch nodes with multiprocessing worker concurrency. - * cobalt_submit_mproc.sh: Example submission script to run libEnsemble similarly on ALCF's Theta, a now-retired system. diff --git a/libensemble/tests/scaling_tests/forces/forces_app/build_forces.sh b/libensemble/tests/scaling_tests/forces/forces_app/build_forces.sh index 7280ae9982..b8b379e0ee 100755 --- a/libensemble/tests/scaling_tests/forces/forces_app/build_forces.sh +++ b/libensemble/tests/scaling_tests/forces/forces_app/build_forces.sh @@ -36,19 +36,19 @@ mpicc -O3 -o forces.x forces.c -lm # Building with OpenMP for target device (e.g. GPU) # ------------------------------------------------- -# xl (plain and using mpicc on Summit) -# xlc_r -DGPU -O3 -qsmp=omp -qoffload -o forces.x forces.c -# mpicc -DGPU -O3 -qsmp=omp -qoffload -o forces.x forces.c +# Aurora: Intel oneAPI (Clang based) Compiler (JIT compiled for device) +# mpicc -DGPU -O3 -fiopenmp -fopenmp-targets=spir64 -o forces.x forces.c -# Summit with gcc (Need up to offload capable gcc: module load gcc/12.1.0) - slower than xlc -# mpicc -DGPU -Ofast -fopenmp -Wl,-rpath=/sw/summit/gcc/12.1.0-0/lib64 -lm -foffload=nvptx-none forces.c -o forces.x +# Frontier (AMD ROCm compiler) +# cc -DGPU -I${ROCM_PATH}/include -L${ROCM_PATH}/lib -lamdhip64 -fopenmp -O3 -o forces.x forces.c # Nvidia (nvc) compiler with mpicc and on Cray system with target (Perlmutter) # mpicc -DGPU -O3 -fopenmp -mp=gpu -o forces.x forces.c # cc -DGPU -Wl,-znoexecstack -O3 -fopenmp -mp=gpu -target-accel=nvidia80 -o forces.x forces.c -# Spock/Crusher/Frontier (AMD ROCm compiler) -# cc -DGPU -I${ROCM_PATH}/include -L${ROCM_PATH}/lib -lamdhip64 -fopenmp -O3 -o forces.x forces.c +# xl (plain and using mpicc on Summit) +# xlc_r -DGPU -O3 -qsmp=omp -qoffload -o forces.x forces.c +# mpicc -DGPU -O3 -qsmp=omp -qoffload -o forces.x forces.c -# Aurora: Intel oneAPI (Clang based) Compiler (JIT compiled for device) -# mpicc -DGPU -O3 -fiopenmp -fopenmp-targets=spir64 -o forces.x forces.c +# Summit with gcc (Need up to offload capable gcc: module load gcc/12.1.0) - slower than xlc +# mpicc -DGPU -Ofast -fopenmp -Wl,-rpath=/sw/summit/gcc/12.1.0-0/lib64 -lm -foffload=nvptx-none forces.c -o forces.x diff --git a/libensemble/tests/scaling_tests/forces/forces_multi_app/forces_simf.py b/libensemble/tests/scaling_tests/forces/forces_multi_app/forces_simf.py index bebc748172..3fbe03f2da 100644 --- a/libensemble/tests/scaling_tests/forces/forces_multi_app/forces_simf.py +++ b/libensemble/tests/scaling_tests/forces/forces_multi_app/forces_simf.py @@ -8,9 +8,8 @@ def run_forces(H, persis_info, sim_specs, libE_info): - """Launches the forces MPI app and auto-assigns ranks and GPU resources. - - Assigns one MPI rank to each GPU assigned to the worker. + """Launches the either the CPU-only or GPU version of the forces MPI app + and auto-assigns ranks and GPU resources as requested by the generator. """ calc_status = 0 diff --git a/libensemble/tests/scaling_tests/forces/forces_multi_app/run_libe_forces.py b/libensemble/tests/scaling_tests/forces/forces_multi_app/run_libe_forces.py index 235aa92279..a55d502ead 100644 --- a/libensemble/tests/scaling_tests/forces/forces_multi_app/run_libe_forces.py +++ b/libensemble/tests/scaling_tests/forces/forces_multi_app/run_libe_forces.py @@ -16,10 +16,10 @@ ((nworkers - 1) - gpus_on_node) >= gen_specs["user"][max_procs] E.g., if running on one node with four GPUs, then use: - python run_libE_forces.py --comms local --nworkers 9 + python run_libE_forces.py --nworkers 9 E.g., if running on one node with eight GPUs, then use: - python run_libE_forces.py --comms local --nworkers 17 + python run_libE_forces.py --nworkers 17 """ import os diff --git a/libensemble/tests/scaling_tests/forces/forces_simple/forces_simf.py b/libensemble/tests/scaling_tests/forces/forces_simple/forces_simf.py index ecc2ad4b53..74e2ac2dbd 100644 --- a/libensemble/tests/scaling_tests/forces/forces_simple/forces_simf.py +++ b/libensemble/tests/scaling_tests/forces/forces_simple/forces_simf.py @@ -5,7 +5,14 @@ def run_forces(H, persis_info, sim_specs, libE_info): - """Runs the forces MPI application""" + """Runs the forces MPI application. + + By default assigns the number of MPI ranks to the number + of cores available to this worker. + + To assign a different number give e.g., `num_procs=4` to + ``exctr.submit``. + """ calc_status = 0 diff --git a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_aurora.sh b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_aurora.sh index fcc3ef80e9..3e5e864fd0 100644 --- a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_aurora.sh +++ b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_aurora.sh @@ -4,15 +4,14 @@ #PBS -q #PBS -A -module use /soft/modulefiles module load frameworks export MPICH_GPU_SUPPORT_ENABLED=1 cd $PBS_O_WORKDIR # 2 nodes - 12 sim workers (6 GPUs per node) -python run_libe_forces.py --comms local --nworkers 13 +python run_libe_forces.py -n 13 # if using libE_specs["use_tiles_as_gpus"] = True # 2 nodes 24 sim workers (12 GPU tiles per node) -# python run_libe_forces.py --comms local --nworkers 25 +# python run_libe_forces.py -n 25 diff --git a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_polaris.sh b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_polaris.sh index 2501b23546..9f5d23f0bb 100644 --- a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_polaris.sh +++ b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_polaris.sh @@ -7,4 +7,4 @@ export MPICH_GPU_SUPPORT_ENABLED=1 cd $PBS_O_WORKDIR -python run_libe_forces.py --comms local --nworkers 5 +python run_libe_forces.py -n 5 diff --git a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_simple.sh b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_simple.sh index 2501b23546..9f5d23f0bb 100644 --- a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_simple.sh +++ b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_pbs_simple.sh @@ -7,4 +7,4 @@ export MPICH_GPU_SUPPORT_ENABLED=1 cd $PBS_O_WORKDIR -python run_libe_forces.py --comms local --nworkers 5 +python run_libe_forces.py -n 5 diff --git a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_slurm_simple.sh b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_slurm_simple.sh index 97b600ac3e..b59bf94f14 100644 --- a/libensemble/tests/scaling_tests/forces/submission_scripts/submit_slurm_simple.sh +++ b/libensemble/tests/scaling_tests/forces/submission_scripts/submit_slurm_simple.sh @@ -12,4 +12,4 @@ export SLURM_EXACT=1 export SLURM_MEM_PER_NODE=0 -python run_libe_forces.py --comms local --nworkers 9 +python run_libe_forces.py -n 9 diff --git a/libensemble/tests/scaling_tests/persistent_gp/run_example.py b/libensemble/tests/scaling_tests/persistent_gp/run_example.py index 5f7d642802..035fa9c1e0 100644 --- a/libensemble/tests/scaling_tests/persistent_gp/run_example.py +++ b/libensemble/tests/scaling_tests/persistent_gp/run_example.py @@ -3,7 +3,7 @@ Usage: ------ -python run_example.py --comms local --nworkers 4 +python run_example.py --nworkers 4 """ import numpy as np diff --git a/libensemble/tests/scaling_tests/warpx/all_machine_specs.py b/libensemble/tests/scaling_tests/warpx/all_machine_specs.py index dc3ecaca92..56a0623398 100644 --- a/libensemble/tests/scaling_tests/warpx/all_machine_specs.py +++ b/libensemble/tests/scaling_tests/warpx/all_machine_specs.py @@ -14,13 +14,3 @@ "OMP_NUM_THREADS": "2", "sim_max": 3, # Maximum number of simulations } - - -summit_specs = { - "name": "summit", # Machine name - "sim_app": os.environ["HOME"] + "/warpx/Bin/main2d.gnu.TPROF.MPI.CUDA.ex", - # extra arguments passed to jsrun at execution - "extra_args": '-n 1 -a 1 -g 1 -c 1 --bind=packed:1 --smpiargs="-gpu"', - "OMP_NUM_THREADS": "1", - "sim_max": 400, # Maximum number of simulations -} diff --git a/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py b/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py index 636767c0ed..fa0fbc2165 100644 --- a/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py +++ b/libensemble/tests/scaling_tests/warpx/run_libensemble_on_warpx.py @@ -6,9 +6,7 @@ then launches WarpX simulations. Execute locally via the following command: - python run_libensemble_on_warpx.py --comms local --nworkers 3 -On summit, use the submission script: - bsub summit_submit_mproc.sh + python run_libensemble_on_warpx.py --nworkers 3 The number of concurrent evaluations of the objective function will be nworkers=1 as one worker is for the persistent gen_f. @@ -16,7 +14,6 @@ # Either 'random' or 'aposmm' generator_type = "aposmm" -# Either 'local' or 'summit' machine = "local" import sys @@ -49,8 +46,6 @@ # Import machine-specific run parameters if machine == "local": machine_specs = all_machine_specs.local_specs -elif machine == "summit": - machine_specs = all_machine_specs.summit_specs else: print("you shouldn' hit that") sys.exit() diff --git a/libensemble/tests/scaling_tests/warpx/warpx_simf.py b/libensemble/tests/scaling_tests/warpx/warpx_simf.py index a3852a99e9..6d88d05433 100644 --- a/libensemble/tests/scaling_tests/warpx/warpx_simf.py +++ b/libensemble/tests/scaling_tests/warpx/warpx_simf.py @@ -44,25 +44,14 @@ def run_warpx(H, persis_info, sim_specs, libE_info): app_args = input_file os.environ["OMP_NUM_THREADS"] = machine_specs["OMP_NUM_THREADS"] - # Launch the executor to actually run the WarpX simulation - if machine_specs["name"] == "summit": - task = exctr.submit( - app_name="warpx", - extra_args=machine_specs["extra_args"], - app_args=app_args, - stdout="out.txt", - stderr="err.txt", - wait_on_start=True, - ) - else: - task = exctr.submit( - app_name="warpx", - num_procs=machine_specs["cores"], - app_args=app_args, - stdout="out.txt", - stderr="err.txt", - wait_on_start=True, - ) + task = exctr.submit( + app_name="warpx", + num_procs=machine_specs["cores"], + app_args=app_args, + stdout="out.txt", + stderr="err.txt", + wait_on_start=True, + ) # Periodically check the status of the simulation poll_interval = 1 # secs diff --git a/libensemble/tests/unit_tests/test_platform.py b/libensemble/tests/unit_tests/test_platform.py index ba27d03b2d..238631448a 100644 --- a/libensemble/tests/unit_tests/test_platform.py +++ b/libensemble/tests/unit_tests/test_platform.py @@ -15,13 +15,13 @@ "cores_per_node": 32, } -summit_spec = { - "mpi_runner": "jsrun", - "cores_per_node": 42, - "logical_cores_per_node": 168, - "gpus_per_node": 6, - "gpu_setting_type": "option_gpus_per_task", - "gpu_setting_name": "-g", +frontier_spec = { + "mpi_runner": "srun", + "cores_per_node": 64, + "logical_cores_per_node": 128, + "gpus_per_node": 8, + "gpu_setting_type": "runner_default", + "gpu_env_fallback": "ROCR_VISIBLE_DEVICES", "scheduler_match_slots": False, } @@ -101,10 +101,10 @@ def test_known_sys_detect(monkeypatch): monkeypatch.delenv("NERSC_HOST", raising=False) known_platforms = specs_dump(Known_platforms(), exclude_none=True) - get_sys_cmd = "echo summit.olcf.ornl.gov" # Overrides default "hostname -d" + get_sys_cmd = "echo frontier.olcf.ornl.gov" # Overrides default "hostname -d" name = known_system_detect(cmd=get_sys_cmd) platform_info = known_platforms[name] - assert platform_info == summit_spec, f"Summit spec does not match expected ({platform_info})" + assert platform_info == frontier_spec, f"Frontier spec does not match expected ({platform_info})" # Try unknown system get_sys_cmd = "echo madeup.system" # Overrides default "hostname -d" diff --git a/libensemble/tests/unit_tests/test_ufunc_runners.py b/libensemble/tests/unit_tests/test_ufunc_runners.py index 1d3cbb4b2c..09f17b07ec 100644 --- a/libensemble/tests/unit_tests/test_ufunc_runners.py +++ b/libensemble/tests/unit_tests/test_ufunc_runners.py @@ -54,6 +54,20 @@ def tupilize(arg1, arg2): simrunner.shutdown() +def test_persis_info_from_none(): + calc_in, sim_specs, gen_specs = get_ufunc_args() + + def tupilize(arg1, arg2): + return (arg1, arg2) + + sim_specs["sim_f"] = tupilize + simrunner = Runner(sim_specs) + libE_info = {"H_rows": np.array([2, 3, 4]), "workerID": 1, "comm": "fakecomm"} + + result = simrunner.run(calc_in, {"libE_info": libE_info, "persis_info": None, "tag": 1}) + assert result == (calc_in, {}) + + @pytest.mark.extra def test_globus_compute_runner_init(): calc_in, sim_specs, gen_specs = get_ufunc_args() @@ -122,6 +136,7 @@ def test_globus_compute_runner_fail(): if __name__ == "__main__": test_normal_runners() test_thread_runners() + test_persis_info_from_none() test_globus_compute_runner_init() test_globus_compute_runner_pass() test_globus_compute_runner_fail() diff --git a/libensemble/tests/unit_tests_logger/test_logger.py b/libensemble/tests/unit_tests_logger/test_logger.py index 242f5119ce..e06331b3d2 100644 --- a/libensemble/tests/unit_tests_logger/test_logger.py +++ b/libensemble/tests/unit_tests_logger/test_logger.py @@ -5,6 +5,7 @@ """ import logging import os + import pytest from libensemble import logger diff --git a/libensemble/tools/tools.py b/libensemble/tools/tools.py index 5458069673..4038b5fafe 100644 --- a/libensemble/tools/tools.py +++ b/libensemble/tools/tools.py @@ -75,20 +75,21 @@ ) -def _get_shortname(calling_file): - script_name = os.path.splitext(os.path.basename(calling_file))[0] +def _get_shortname(basename): + script_name = os.path.splitext(os.path.basename(basename))[0] short_name = script_name.split("test_", 1).pop() return short_name # =================== save libE output to pickle and np ======================== - -def save_libE_output(H, persis_info, calling_file, nworkers, dest_path=os.getcwd(), mess="Run completed"): +def save_libE_output(H, persis_info, basename, nworkers, dest_path=None, mess="Run completed", append_attrs=True): """ Writes out history array and persis_info to files. - Format: _results_History_length=_evals=_ranks= + Format: _results_History_length=_evals=_ranks= + + To use just basename, set append_attrs=False .. code-block:: python @@ -107,7 +108,7 @@ def save_libE_output(H, persis_info, calling_file, nworkers, dest_path=os.getcwd Persistent information dictionary. :doc:`(example)` - calling_file : :obj:`str` + basename : :obj:`str` Name of user-calling script (or user chosen name) to prefix output files. The convention is to send __file__ from user calling script. @@ -120,12 +121,24 @@ def save_libE_output(H, persis_info, calling_file, nworkers, dest_path=os.getcwd A message to print/log when saving the file. + append_attrs: `bool` + + Append run attributes to the base filename. + """ - short_name = _get_shortname(calling_file) - prob_str = "length=" + str(len(H)) + "_evals=" + str(sum(H["sim_ended"])) + "_workers=" + str(nworkers) + if dest_path is None: + dest_path = os.getcwd() + + short_name = _get_shortname(basename) + + prob_str = hist_name = persis_name = "" + if append_attrs: + prob_str = "length=" + str(len(H)) + "_evals=" + str(sum(H["sim_ended"])) + "_workers=" + str(nworkers) + hist_name = "_history_" + prob_str + persis_name = "_persis_info_" + prob_str - h_filename = os.path.join(dest_path, short_name + "_history_" + prob_str) - p_filename = os.path.join(dest_path, short_name + "_persis_info_" + prob_str) + h_filename = os.path.join(dest_path, short_name + hist_name) + p_filename = os.path.join(dest_path, short_name + persis_name) status_mess = " ".join(["------------------", mess, "-------------------"]) logger.info(f"{status_mess}\nSaving results to file: {h_filename}") diff --git a/libensemble/utils/runners.py b/libensemble/utils/runners.py index 629c733b1b..93a2cd1afe 100644 --- a/libensemble/utils/runners.py +++ b/libensemble/utils/runners.py @@ -37,6 +37,8 @@ def shutdown(self) -> None: pass def run(self, calc_in: npt.NDArray, Work: dict) -> (npt.NDArray, dict, Optional[int]): + if Work["persis_info"] is None: + Work["persis_info"] = {} return self._result(calc_in, Work["persis_info"], Work["libE_info"]) diff --git a/libensemble/version.py b/libensemble/version.py index aa56ed4042..5b60188613 100644 --- a/libensemble/version.py +++ b/libensemble/version.py @@ -1 +1 @@ -__version__ = "1.4.3" +__version__ = "1.5.0" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..c6a7ea9f36 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,138 @@ +[project] + +authors = [{name = "Jeffrey Larson"}, {name = "Stephen Hudson"}, + {name = "Stefan M. Wild"}, {name = "David Bindel"}, + {name = "John-Luke Navarro"}] + +dependencies = [ "numpy", "psutil", "pydantic", "pyyaml", "tomli"] + +description = "A Python toolkit for coordinating asynchronous and dynamic ensembles of calculations." +name = "libensemble" +requires-python = ">=3.10" +license = {file = "LICENSE"} +readme = "README.rst" + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Operating System :: Unix", + "Operating System :: MacOS", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dynamic = ["version"] + +[project.urls] +Documentation = "https://libensemble.readthedocs.io/en/main/" +Repository = "https://github.com/Libensemble/libensemble" +Issues = "https://github.com/Libensemble/libensemble/issues" + +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "wheel", "pip>=24.3.1,<26", "setuptools>=75.1.0,<79", ] + +[tool.setuptools.packages.find] +where = ["."] +include = ["libensemble*"] + +[tool.setuptools.dynamic] +version = {attr = "libensemble.version.__version__"} + +[tool.pixi.project] +channels = ["conda-forge"] +platforms = ["osx-arm64", "linux-64", "osx-64"] + +[tool.pixi.pypi-dependencies] +libensemble = { path = ".", editable = true } + +[tool.pixi.environments] +default = [] +dev = ["dev"] + +[tool.pixi.feature.dev.dependencies] +mpi = ">=1.0.1,<2" +mpich = ">=4.3.0,<5" +mpi4py = ">=4.0.3,<5" +flake8 = ">=7.1.2,<8" +coverage = ">=7.6.12,<8" +pytest = ">=8.3.4,<9" +pytest-cov = ">=6.0.0,<7" +pytest-timeout = ">=2.3.1,<3" +mock = ">=5.1.0,<6" +python-dateutil = ">=2.9.0.post0,<3" +anyio = ">=4.8.0,<5" +matplotlib = ">=3.10.0,<4" +mpmath = ">=1.3.0,<2" +rich = ">=13.9.4,<14" +sphinx = ">=8.2.1,<9" +sphinxcontrib-bibtex = ">=2.6.3,<3" +sphinx-design = ">=0.6.1,<0.7" +sphinx_rtd_theme = ">=3.0.1,<4" +sphinx-copybutton = ">=0.5.2,<0.6" +pre-commit = ">=4.1.0,<5" +nlopt = ">=2.10.0,<3" +scipy = ">=1.15.2,<2" +ax-platform = ">=0.5.0,<0.6" +sphinxcontrib-spelling = ">=8.0.1,<9" +autodoc-pydantic = ">=2.1.0,<3" + +[tool.pixi.dependencies] +python = ">=3.10,<3.14" +pip = ">=24.3.1,<25" +setuptools = ">=75.6.0,<76" +numpy = ">=1.21,<3" +pydantic = ">=1.10,<3" +pyyaml = ">=6.0,<7" +tomli = ">=1.2.1,<3" +psutil = ">=5.9.4,<7" + +[tool.pixi.target.osx-arm64.dependencies] +clang_osx-arm64 = ">=19.1.2,<20" + +[tool.black] +line-length = 120 +target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] +force-exclude = ''' +( + /( + | \.git + | \.github + )/ + | libensemble/files/to/avoid +) +''' + +[tool.typos.default] +extend-ignore-identifiers-re = [ + ".*NDArray.*", + "8ba9de56.*" +] + +[tool.typos.default.extend-words] +als = "als" +datas = "datas" +numer = "numer" +inout = "inout" +arange = "arange" +wrk = "wrk" +EOF = "EOF" +HPE = "HPE" +RO = "RO" +lst = "lst" +noy = "noy" + +[tool.typos.files] +extend-exclude = ["*.bib", "*.xml", "docs/nitpicky"] + +[dependency-groups] +dev = ["pyenchant", "enchant>=0.0.1,<0.0.2"] diff --git a/scripts/aposmm_min_loc.py b/scripts/aposmm_min_loc.py index cc6b1eba62..d0cd48a339 100644 --- a/scripts/aposmm_min_loc.py +++ b/scripts/aposmm_min_loc.py @@ -10,18 +10,19 @@ """ -import numpy as np -import matplotlib.pyplot as plt import glob import os import pickle import sys +import matplotlib.pyplot as plt +import numpy as np + N = 6 # number of opt runs to show. -x_name = 'x0' -y_name = 'x1' -z_name = 'x2' +x_name = "x0" +y_name = "x1" +z_name = "x2" full_bounds = False # For entire input space enter bounds below @@ -35,7 +36,7 @@ # Find the most recent .npy and pickle files try: H_file = max(glob.glob("*.npy"), key=os.path.getmtime) - persis_info_file = max(glob.iglob('*.pickle'), key=os.path.getctime) + persis_info_file = max(glob.iglob("*.pickle"), key=os.path.getctime) except Exception: sys.exit("Need a *.npy and a *.pickle files in run dir. Exiting...") @@ -45,30 +46,30 @@ index_sets = pickle.load(f)["run_order"] # Filter best N opt runs for clearer graph -trimmed_index_sets = {key: indices[:-1] for key, indices in index_sets.items()} -min_f_per_set = [(key, indices, H['f'][indices].min()) for key, indices in trimmed_index_sets.items() if len(indices) > 0] +trim_sets = {key: indices[:-1] for key, indices in index_sets.items()} +min_f_per_set = [(key, indices, H["f"][indices].min()) for key, indices in trim_sets.items() if len(indices) > 0] min_f_per_set_sorted = sorted(min_f_per_set, key=lambda x: x[2])[:N] # Plotting fig = plt.figure(figsize=(6, 6)) -ax = fig.add_subplot(111, projection='3d') +ax = fig.add_subplot(111, projection="3d") for key, indices, _ in min_f_per_set_sorted: - min_f_index = indices[np.argmin(H['f'][indices])] + min_f_index = indices[np.argmin(H["f"][indices])] # Extract the corresponding 3D x position from H try: - x, y, z = H['x'][min_f_index] + x, y, z = H["x"][min_f_index] except ValueError: x = H[x_name][min_f_index] y = H[y_name][min_f_index] z = H[z_name][min_f_index] # Plot the 3D point - ax.scatter(x, y, z, marker='o', s=50, label=f'Opt run {key}') + ax.scatter(x, y, z, marker="o", s=50, label=f"Opt run {key}") # Draw a line from the point to the XY plane (z=0) - ax.plot([x, x], [y, y], [0, z], color='grey', linestyle='--') + ax.plot([x, x], [y, y], [0, z], color="grey", linestyle="--") if full_bounds: ax.set_xlim(x0_min, x0_max) @@ -79,6 +80,6 @@ ax.set_xlabel(x_name) ax.set_ylabel(y_name) ax.set_zlabel(z_name) -ax.set_title('Locations of best points from each optimization run') -ax.legend(bbox_to_anchor=(-0.1, 0.9), loc='upper left', borderaxespad=0) +ax.set_title("Locations of best points from each optimization run") +ax.legend(bbox_to_anchor=(-0.1, 0.9), loc="upper left", borderaxespad=0) plt.savefig(f"location_min_best{N}.png") diff --git a/scripts/aposmm_plot_local_runs.py b/scripts/aposmm_plot_local_runs.py index f94ecb13aa..47a4f8c73b 100755 --- a/scripts/aposmm_plot_local_runs.py +++ b/scripts/aposmm_plot_local_runs.py @@ -9,19 +9,20 @@ """ -import numpy as np -import matplotlib.pyplot as plt import glob import os import pickle import sys +import matplotlib.pyplot as plt +import numpy as np + N = 6 # number of opt runs to show. # Find the most recent .npy and pickle files try: H_file = max(glob.glob("*.npy"), key=os.path.getmtime) - persis_info_file = max(glob.iglob('*.pickle'), key=os.path.getctime) + persis_info_file = max(glob.iglob("*.pickle"), key=os.path.getctime) except Exception: sys.exit("Need a *.npy and a *.pickle files in run dir. Exiting...") @@ -31,38 +32,38 @@ index_sets = pickle.load(f)["run_order"] # Filter best N opt runs for clearer graph -trimmed_index_sets = {key: indices[:-1] for key, indices in index_sets.items()} -min_f_per_set = [(key, indices, H['f'][indices].min()) for key, indices in trimmed_index_sets.items() if len(indices) > 0] +trim_sets = {key: indices[:-1] for key, indices in index_sets.items()} +min_f_per_set = [(key, indices, H["f"][indices].min()) for key, indices in trim_sets.items() if len(indices) > 0] min_f_per_set_sorted = sorted(min_f_per_set, key=lambda x: x[2])[:N] # Plotting plt.figure(figsize=(10, 6)) -plt.scatter(range(len(H['f'])), H['f'], color='lightgrey', label='Other points', zorder=1) +plt.scatter(range(len(H["f"])), H["f"], color="lightgrey", label="Other points", zorder=1) num_runs = len(index_sets) # for key, indices in index_sets.items(): for key, indices, _ in min_f_per_set_sorted: # Extract the 'f' values for each index set - f_values = H['f'][indices] + f_values = H["f"][indices] - ratio = (f_values[-1] - f_values[-2])/ f_values[-2] + ratio = (f_values[-1] - f_values[-2]) / f_values[-2] # print(f"Tolerance of the last two values for {key}: {ratio}") - plt.plot(indices, f_values, marker='o', label=f'Opt run {key}', zorder=2) + plt.plot(indices, f_values, marker="o", label=f"Opt run {key}", zorder=2) # Identify the index and value of the minimum f min_index = indices[np.argmin(f_values)] min_f_value = np.min(f_values) # Mark the minimum f value with a distinct marker - plt.scatter(min_index, min_f_value, color='red', edgecolor='black', s=50, zorder=3) + plt.scatter(min_index, min_f_value, color="red", edgecolor="black", s=50, zorder=3) # Add a dummy point to the legend for "minima of opt run" -plt.scatter([], [], color='red', edgecolor='black', s=50, label='Best value of opt run') +plt.scatter([], [], color="red", edgecolor="black", s=50, label="Best value of opt run") -plt.xlabel('Simulation ID') -plt.ylabel('Objective value') -plt.title(f'Objective values by Optimization runs. Best {N} runs from {num_runs}') +plt.xlabel("Simulation ID") +plt.ylabel("Objective value") +plt.title(f"Objective values by Optimization runs. Best {N} runs from {num_runs}") plt.legend() plt.grid(True) plt.savefig(f"opt_runs_best{N}.png") diff --git a/setup.py b/setup.py index 72e2c2a33e..606849326a 100644 --- a/setup.py +++ b/setup.py @@ -1,115 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# This is for setting up libEnsemble, license and details can be -# found at https://github.com/Libensemble/libensemble/ - -"""libEnsemble - -libEnsemble is a Python toolkit for coordinating workflows of asynchronous -and dynamic ensembles of calculations. - -libEnsemble can help users take advantage of massively parallel resources to -solve design, decision, and inference problems and expand the class of -problems that can benefit from increased parallelism. - -""" - -from pathlib import Path - from setuptools import setup -from setuptools.command.test import test as TestCommand - -exec(open("libensemble/version.py").read()) - - -class Run_TestSuite(TestCommand): - def run_tests(self): - import os - import sys - - py_version = sys.version_info[0] - print("Python version from setup.py is", py_version) - run_string = "libensemble/tests/run-tests.sh -p " + str(py_version) - os.system(run_string) - - -class ToxTest(TestCommand): - user_options = [] - - def initialize_options(self): - TestCommand.initialize_options(self) - - def run_tests(self): - import tox - - tox.cmdline() - -setup( - name="libensemble", - version=__version__, - description="Library to coordinate the concurrent evaluation of dynamic ensembles of calculations", - long_description=Path("README.rst").read_text(encoding="utf-8"), - url="https://github.com/Libensemble/libensemble", - author="Jeffrey Larson, Stephen Hudson, Stefan M. Wild, David Bindel and John-Luke Navarro", - author_email="libensemble@lists.mcs.anl.gov", - license="BSD 3-clause", - packages=[ - "libensemble", - "libensemble.gen_funcs", - "libensemble.sim_funcs", - "libensemble.sim_funcs.branin", - "libensemble.alloc_funcs", - "libensemble.tests", - "libensemble.comms", - "libensemble.utils", - "libensemble.tools", - "libensemble.tools.live_data", - "libensemble.executors", - "libensemble.resources", - "libensemble.tests.unit_tests", - "libensemble.tests.regression_tests", - ], - install_requires=["numpy>=1.21", "psutil>=5.9.4", "pydantic>=1.10", "tomli>=1.2.1", "pyyaml>=6.0"], - # numpy - oldest working version. psutil - oldest working version. - # pyyaml - oldest working version. - # If run tests through setup.py - downloads these but does not install - tests_require=[ - "pytest>=3.1", - "pytest-cov>=2.5", - "pytest-pep8>=1.0", - "pytest-timeout", - "mock", - ], - extras_require={ - "docs": [ - "autodoc_pydantic", - "sphinx<9", - "sphinx_design", - "sphinx_rtd_theme", - "sphinxcontrib-bibtex", - "sphinxcontrib-spelling", - "sphinx-copybutton", - ], - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - "Operating System :: MacOS", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - cmdclass={"test": Run_TestSuite, "tox": ToxTest}, -) +setup()