diff --git a/.clang-format b/.clang-format index b3bb8d132..6f7fef667 100644 --- a/.clang-format +++ b/.clang-format @@ -108,6 +108,10 @@ IncludeCategories: Priority: 4 - Regex: '^"checkpoint\/.*\.h"' Priority: 4 + - Regex: '^"kernels\/.*\.hpp"' + Priority: 4 + - Regex: '^"kernels\/.*\.h"' + Priority: 4 - Regex: '^"output\/.*\.h"' Priority: 4 - Regex: '^"archetypes\/.*\.h"' diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..262274baf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/cpuarch.yml b/.github/workflows/cpuarch.yml index 672d26395..2c68e984f 100644 --- a/.github/workflows/cpuarch.yml +++ b/.github/workflows/cpuarch.yml @@ -2,6 +2,8 @@ name: CPU Compilation/Unit Tests on: push: + pull_request: + types: [ready_for_review] jobs: check-commit: @@ -14,6 +16,10 @@ jobs: - name: Check commit message id: check_message run: | + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.action }}" == "ready_for_review" ]]; then + echo "run_tests=true" >> "$GITHUB_OUTPUT" + exit 0 + fi if git log -1 --pretty=%B | grep -q "CPUTEST"; then echo "run_tests=true" >> "$GITHUB_OUTPUT" else @@ -57,8 +63,10 @@ jobs: cmake --version - name: Install OpenMPI 5 run: | - sudo apt-get install -y libopenmpi-dev openmpi-bin - mpirun --version + if [ "${{ matrix.mpi }}" = "ON" ]; then + sudo apt-get install -y libopenmpi-dev openmpi-bin + mpirun --version + fi - name: Configure run: | if [ "${{ matrix.mpi }}" = "ON" ]; then @@ -81,7 +89,23 @@ jobs: strategy: fail-fast: false matrix: - pgen: [streaming, turbulence, reconnection, shock, magnetosphere, accretion, wald] + pgen: + [ + streaming, + turbulence, + reconnection, + shock, + magnetosphere, + accretion, + wald, + examples/custom_emission, + examples/external_fields, + examples/match_fix_field_boundaries, + examples/custom_energy_distribution, + examples/custom_spatial_distribution, + examples/atmosphere, + examples/replenish_injector, + ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 52d372f40..62248c605 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ venv/ *.mov *.mp4 !pgens/**/*.png +!pgens/**/*.mp4 # Accidental files *.xc @@ -59,3 +60,4 @@ tags action-token *.vim ignore-* +tombi/ diff --git a/.gitmodules b/.gitmodules index 835fbe5b8..f66f13275 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "extern/plog"] - path = extern/plog - url = https://github.com/SergiusTheBest/plog.git [submodule "extern/adios2"] path = extern/adios2 url = https://github.com/ornladios/ADIOS2.git diff --git a/CMakeLists.txt b/CMakeLists.txt index fad95b106..985eebc53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(PROJECT_NAME entity) project( ${PROJECT_NAME} - VERSION 1.3.3 + VERSION 1.4.0 LANGUAGES CXX C) add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"") set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ") @@ -47,9 +47,6 @@ set(pgen ${default_pgen} CACHE STRING "Problem generator") -set(gui - ${default_gui} - CACHE BOOL "Use GUI [nttiny]") set(output ${default_output} CACHE BOOL "Enable output") @@ -62,7 +59,7 @@ set(gpu_aware_mpi CACHE BOOL "Enable GPU-aware MPI") # -------------------------- Compilation settings -------------------------- # -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -70,7 +67,7 @@ if(${DEBUG} STREQUAL "OFF") set(CMAKE_BUILD_TYPE Release CACHE STRING "CMake build type") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG -O3") else() set(CMAKE_BUILD_TYPE Debug @@ -109,9 +106,8 @@ set(BUILD_TESTING include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/dependencies.cmake) find_or_fetch_dependency(Kokkos FALSE QUIET) -find_or_fetch_dependency(plog TRUE QUIET) set(DEPENDENCIES Kokkos::kokkos) -include_directories(${plog_SRC}/include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) # -------------------------------- Main code ------------------------------- # set_precision(${precision}) @@ -162,29 +158,37 @@ if(${output}) find_or_fetch_dependency(adios2 FALSE QUIET) add_compile_options("-D OUTPUT_ENABLED") if(${mpi}) - set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11_mpi) + set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx_mpi) else() - set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx11) + set(DEPENDENCIES ${DEPENDENCIES} adios2::cxx) endif() endif() link_libraries(${DEPENDENCIES}) +set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/) +add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global) +add_subdirectory(${SRC_DIR}/metrics ${CMAKE_CURRENT_BINARY_DIR}/metrics) +add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels) +add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes) +add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework) +add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output) + +# ------------------------------- Main source ------------------------------ # +if(NOT ${pgen} STREQUAL ${default_pgen}) + set_problem_generator(${pgen}) + add_subdirectory(${SRC_DIR}/engines ${CMAKE_CURRENT_BINARY_DIR}/engines) + add_subdirectory(${SRC_DIR} ${CMAKE_CURRENT_BINARY_DIR}/src) +endif() + if(TESTS) # ---------------------------------- Tests --------------------------------- # include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests.cmake) -elseif(BENCHMARK) +endif() + +if(BENCHMARK) # ------------------------------ Benchmark --------------------------------- # include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/benchmark.cmake) -else() - # ----------------------------------- GUI ---------------------------------- # - if(${gui}) - find_or_fetch_dependency(nttiny FALSE QUIET) - endif() - - # ------------------------------- Main source ------------------------------ # - set_problem_generator(${pgen}) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src src) endif() include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/report.cmake) diff --git a/README.md b/README.md index 41d5ae280..10d1634dd 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ tl;dr: One particle-in-cell code to rule them all. -`Entity` is a community-driven open-source coordinate-agnostic general-relativistic (GR) particle-in-cell (PIC) code written in C++17 specifically targeted to study plasma physics in relativistic astrophysical systems. The main algorithms of the code are written in covariant form, allowing to easily implement arbitrary grid geometries. The code is highly modular, and is written in the architecture-agnostic way using the [`Kokkos`](https://kokkos.org/kokkos-core-wiki/) performance portability library, allowing the code to efficiently use device parallelization on CPU and GPU architectures of different types. The multi-node parallelization is implemented using the `MPI` library, and the data output is done via the [`ADIOS2`](https://github.com/ornladios/ADIOS2) library which supports multiple output formats, including `HDF5` and `BP5`. +`Entity` is a community-driven open-source coordinate-agnostic general-relativistic (GR) particle-in-cell (PIC) code written in C++ specifically targeted to study plasma physics in relativistic astrophysical systems. The main algorithms of the code are written in covariant form, allowing to easily implement arbitrary grid geometries. The code is highly modular, and is written in the architecture-agnostic way using the [`Kokkos`](https://kokkos.org/kokkos-core-wiki/) performance portability library, allowing the code to efficiently use device parallelization on CPU and GPU architectures of different types. The multi-node parallelization is implemented using the `MPI` library, and the data output is done via the [`ADIOS2`](https://github.com/ornladios/ADIOS2) library which supports multiple output formats, including `HDF5` and `BP5`. `Entity` is part of the `Entity toolkit` framework, which also includes a Python library for fast and efficient data analysis and visualization of the simulation data: [`nt2py`](https://pypi.org/project/nt2py/). @@ -12,6 +12,10 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth ## News +- [Mar 2026]: **single-particle emission** [PR #174](https://github.com/entity-toolkit/entity/pull/174) and [PR #188](https://github.com/entity-toolkit/entity/pull/188) +- [Mar 2026]: **spatial sorting of particles** [PR #181](https://github.com/entity-toolkit/entity/pull/181) +- [Mar 2026]: **external EM & force fields** [PR #183](https://github.com/entity-toolkit/entity/pull/183) +- [Mar 2026]: **examples** of most commonly used custom patterns (see `pgens/examples`) - [Dec 2025]: **high-order** shape functions [PR #109](https://github.com/entity-toolkit/entity/pull/109) and advanced field stencils [PR #103](https://github.com/entity-toolkit/entity/pull/103) are now supported. - [Dec 2025]: **particle tracking** is now fully supported via [PR #144](https://github.com/entity-toolkit/entity/pull/144). - [Nov 2025]: our **method papers** are online: [Special relativistic module](https://ui.adsabs.harvard.edu/abs/2025arXiv251117710H/abstract), [GR module](https://ui.adsabs.harvard.edu/abs/2025arXiv251117701G/abstract)! @@ -24,7 +28,7 @@ Please, see the `CITATION` document for the relevant BibTeX entries if you would Everyone is welcome to join our small yet steadily growing community of code users and developers; regardless of how much you are planning to contribute -- we always welcome fresh ideas and feedback. We hold weekly Slack calls on Mondays at 12pm NY time, and have a dedicated Slack channel where you can be easily added by emailing one of the maintainers (indicated with an asterisk in the list below). Anyone is welcome to join both our **Slack workspace** and the weekly meetings -- please feel free to request access by emailing. -Another way of contacting us is via GitHub issues and/or pull requests. Make sure to check out our [F.A.Q.](https://entity-toolkit.github.io/wiki/content/1-getting-started/9-faq/), as it might help you answer your question. +Another way of contacting us is via GitHub issues and/or pull requests. Make sure to check out our [F.A.Q.](https://entity-toolkit.github.io/wiki/content/2-howto/7-faq/), as it might help you answer your question. > Keep in mind, you are free to use the code in any capacity, and there is absolutely no requirement on our end of including any of the developers in your project/proposal (as highlighted in our Code of Conduct). When contributing, also keep in mind that the code you upload to the repository automatically becomes public and open-source, and the same standards will be applied to it as to the rest of the code. @@ -37,11 +41,16 @@ Maintainers indicated with an arrow. * :tipping_hand_person: Alexander Chernoglazov {[@SChernoglazov](https://github.com/SChernoglazov)} * :tea: Benjamin Crinquand {[@bcrinquand](https://github.com/bcrinquand)} * :bubble_tea: Alisa Galishnikova {[@alisagk](https://github.com/alisagk)} -* :steam_locomotive: Evgeny Gorbunov {[@Alcauchy](https://github.com/Alcauchy)} [-> [haykh.astro [at] gmail](mailto:haykh.astro@gmail.com)] -* :coffee: Hayk Hakobyan {[@haykh](https://github.com/haykh)} [-> [genegorbs [at] gmail](mailto:genegorbs@gmail.com)] +* :sloth: Xingwei Gong {[@xwgong01](https://github.com/xwgong01)} +* :steam_locomotive: Evgeny Gorbunov {[@Alcauchy](https://github.com/Alcauchy)} [-> [genegorbs [at] gmail](mailto:genegorbs@gmail.com)] +* :ant: Camille Granier {[@K1000Granier](https://github.com/K1000Granier)} +* :fried_egg: Michael Grehan {[@mgrehan](https://github.com/mgrehan)} +* :coffee: Hayk Hakobyan {[@haykh](https://github.com/haykh)} [-> [haykh.astro [at] gmail](mailto:haykh.astro@gmail.com)] +* :sunrise_over_mountains: Anuj Kankani {[@AnujKankani](https://github.com/AnujKankani)} * :potato: Jens Mahlmann {[@jmahlmann](https://github.com/jmahlmann)} * :dolphin: Sasha Philippov {[@sashaph](https://github.com/sashaph)} * :radio: Siddhant Solanki {[@sidruns30](https://github.com/sidruns30)} +* :mango: Andrew Sullivan {[@a-sullivan](https://github.com/a-sullivan)} * :shrug: Arno Vanthieghem {[@vanthieg](https://github.com/vanthieg)} * :cat: Muni Zhou {[@munizhou](https://github.com/munizhou)} diff --git a/cmake/benchmark.cmake b/cmake/benchmark.cmake index fdd8438ea..af0d334ea 100644 --- a/cmake/benchmark.cmake +++ b/cmake/benchmark.cmake @@ -2,17 +2,6 @@ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) -add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global) -add_subdirectory(${SRC_DIR}/metrics ${CMAKE_CURRENT_BINARY_DIR}/metrics) -add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels) -add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes) -add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework) - -if(${output}) - add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output) - add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint) -endif() - set(exec benchmark.xc) set(src ${CMAKE_CURRENT_SOURCE_DIR}/benchmark/benchmark.cpp) @@ -23,4 +12,7 @@ if(${output}) list(APPEND libs ntt_output) endif() add_dependencies(${exec} ${libs}) -target_link_libraries(${exec} PRIVATE ${libs} stdc++fs) +if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "AppleClang") + list(APPEND libs stdc++fs) +endif() +target_link_libraries(${exec} PRIVATE ${libs}) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 1f3ed3c6a..2024702b5 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -3,14 +3,24 @@ set(Kokkos_REPOSITORY https://github.com/kokkos/kokkos.git CACHE STRING "Kokkos repository") -set(plog_REPOSITORY - https://github.com/SergiusTheBest/plog.git - CACHE STRING "plog repository") +set(Kokkos_TAG + 5.0.1 + CACHE STRING "Kokkos tag") set(adios2_REPOSITORY https://github.com/ornladios/ADIOS2.git CACHE STRING "ADIOS2 repository") +set(adios2_TAG + v2.11.0 + CACHE STRING "ADIOS2 tag") + +set(CONNECTION_CHECKED + FALSE + CACHE BOOL "Whether internet connection has been checked") function(check_internet_connection) + if(CONNECTION_CHECKED) + return() + endif() if(OFFLINE STREQUAL "ON") set(FETCHCONTENT_FULLY_DISCONNECTED ON @@ -35,6 +45,9 @@ function(check_internet_connection) message(STATUS "${Green}Internet connection established.${ColorReset}") endif() endif() + set(CONNECTION_CHECKED + TRUE + CACHE BOOL "Whether internet connection has been checked") endfunction() function(find_or_fetch_dependency package_name header_only mode) @@ -43,6 +56,8 @@ function(find_or_fetch_dependency package_name header_only mode) endif() if(NOT ${package_name}_FOUND) + check_internet_connection() + if(${package_name} STREQUAL "Kokkos") include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkosConfig.cmake) elseif(${package_name} STREQUAL "adios2") @@ -54,16 +69,11 @@ function(find_or_fetch_dependency package_name header_only mode) message(STATUS "${Blue}${package_name} not found. " "Fetching from ${${package_name}_REPOSITORY}${ColorReset}") include(FetchContent) - if(${package_name} STREQUAL "Kokkos") + if(${package_name} STREQUAL "Kokkos" OR ${package_name} STREQUAL "adios2") FetchContent_Declare( ${package_name} GIT_REPOSITORY ${${package_name}_REPOSITORY} - GIT_TAG 4.7.01) - elseif(${package_name} STREQUAL "adios2") - FetchContent_Declare( - ${package_name} - GIT_REPOSITORY ${${package_name}_REPOSITORY} - GIT_TAG v2.10.2) + GIT_TAG ${${package_name}_TAG}) else() FetchContent_Declare(${package_name} GIT_REPOSITORY ${${package_name}_REPOSITORY}) @@ -171,5 +181,3 @@ function(find_or_fetch_dependency package_name header_only mode) ${${package_name}_BUILD_DIR} PARENT_SCOPE) endfunction() - -check_internet_connection() diff --git a/cmake/report.cmake b/cmake/report.cmake index 8f62ac17b..0fbd072cf 100644 --- a/cmake/report.cmake +++ b/cmake/report.cmake @@ -8,7 +8,9 @@ if(${PGEN_FOUND}) "${Blue}" PGEN_REPORT 0) -elseif(${TESTS}) +endif() + +if(${TESTS}) set(TEST_NAMES "") foreach(test_dir IN LISTS TEST_DIRECTORIES) get_property( @@ -18,14 +20,38 @@ elseif(${TESTS}) list(APPEND TEST_NAMES ${LOCAL_TEST_NAMES}) endforeach() printchoices( - "Test cases" + "Tests" + "TESTS" + "${ON_OFF_VALUES}" + "ON" + "OFF" + "${Green}" + TESTS_REPORT_1 + 46) + printchoices( + "" "" "${TEST_NAMES}" "" "${ColorReset}" "" - TESTS_REPORT + TESTS_REPORT_2 0) + # remove only first line of TESTS_REPORT_2 + string(REPLACE "\n" ";" TESTS_REPORT_2_LIST "${TESTS_REPORT_2}") + list(REMOVE_AT TESTS_REPORT_2_LIST 0) + string(REPLACE ";" "\n" TESTS_REPORT_2 "${TESTS_REPORT_2_LIST}") + set(TESTS_REPORT "${TESTS_REPORT_1}\n${TESTS_REPORT_2}") +else() + printchoices( + "Tests" + "TESTS" + "${ON_OFF_VALUES}" + "OFF" + "OFF" + "${Green}" + TESTS_REPORT + 46) endif() printchoices( @@ -120,9 +146,8 @@ string(APPEND REPORT_TEXT ${DASHED_LINE_SYMBOL} "\n" "Configurations" "\n") if(${PGEN_FOUND}) string(APPEND REPORT_TEXT " " ${PGEN_REPORT} "\n") -elseif(${TESTS}) - string(APPEND REPORT_TEXT " " ${TESTS_REPORT} "\n") endif() +string(APPEND REPORT_TEXT " " ${TESTS_REPORT} "\n") string( APPEND diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 0eb043f70..bc397ea53 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -3,25 +3,13 @@ enable_testing() set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) -add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global) -add_subdirectory(${SRC_DIR}/metrics ${CMAKE_CURRENT_BINARY_DIR}/metrics) -add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels) -add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes) -add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework) -add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output) - set(TEST_DIRECTORIES "") -if(NOT ${mpi}) - list(APPEND TEST_DIRECTORIES global) - list(APPEND TEST_DIRECTORIES metrics) - list(APPEND TEST_DIRECTORIES kernels) - list(APPEND TEST_DIRECTORIES archetypes) - list(APPEND TEST_DIRECTORIES framework) -elseif(${mpi} AND ${output}) - list(APPEND TEST_DIRECTORIES framework) -endif() - +list(APPEND TEST_DIRECTORIES global) +list(APPEND TEST_DIRECTORIES metrics) +list(APPEND TEST_DIRECTORIES kernels) +list(APPEND TEST_DIRECTORIES archetypes) +list(APPEND TEST_DIRECTORIES framework) list(APPEND TEST_DIRECTORIES output) foreach(test_dir IN LISTS TEST_DIRECTORIES) diff --git a/dependencies.py b/dependencies.py new file mode 100755 index 000000000..e6bf5b2f7 --- /dev/null +++ b/dependencies.py @@ -0,0 +1,1198 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import curses +import json +import os +from dataclasses import dataclass, field +from typing import Callable, List, Optional, Tuple + + +# ============================ +# colors: edit these +# ============================ + +# foreground colors (use curses.COLOR_* or -1 for default) +COLOR_TITLE_FG = curses.COLOR_BLUE +COLOR_TEXT_FG = curses.COLOR_WHITE +COLOR_SELECTED_FG = curses.COLOR_WHITE +COLOR_SELECTED_BG = curses.COLOR_BLACK +COLOR_HINT_FG = curses.COLOR_YELLOW +COLOR_OK_FG = curses.COLOR_GREEN +COLOR_ERR_FG = curses.COLOR_RED +COLOR_KEY_FG = curses.COLOR_MAGENTA +COLOR_DIM_FG = curses.COLOR_CYAN + +# pair IDs (must be unique small ints) +PAIR_TITLE = 1 +PAIR_TEXT = 2 +PAIR_SELECTED = 3 +PAIR_HINT = 4 +PAIR_OK = 5 +PAIR_ERR = 6 +PAIR_KEY = 7 +PAIR_DIM = 8 + + +KOKKOS_BACKENDS = ["cpu", "cuda", "hip", "sycl"] +ADIOS2_MPI_MODES = ["non-mpi", "mpi"] + +MESSAGE: str = "" + + +@dataclass +class Settings: + cluster: str = "(custom)" + write_modulefiles: bool = False + overwrite: bool = False + install_prefix: str = os.path.join(os.path.expanduser("~"), ".entity") + + apps: dict = field( + default_factory=lambda: {"Kokkos": False, "adios2": False, "nt2py": False} + ) + + # versions + kokkos_version: str = "5.0.1" + adios2_version: str = "2.11.0" + + # options + kokkos_backend: str = "cpu" + kokkos_arch: str = "" + extra_kokkos_flags: List[str] = field(default_factory=list) + adios2_mpi: str = "non-mpi" + extra_adios2_flags: List[str] = field(default_factory=list) + + module_loads: List[str] = field(default_factory=list) + + def from_json(self, json_str: str) -> None: + data = json.loads(json_str) + self.cluster = data.get("cluster", self.cluster) + self.write_modulefiles = data.get("write_modulefiles", self.write_modulefiles) + self.overwrite = data.get("overwrite", self.overwrite) + self.install_prefix = data.get("install_prefix", self.install_prefix) + self.apps = data.get("dependencies", self.apps) + versions = data.get("versions", {}) + self.kokkos_version = versions.get("Kokkos", self.kokkos_version) + self.adios2_version = versions.get("adios2", self.adios2_version) + options = data.get("options", {}) + self.kokkos_backend = options.get("kokkos_backend", self.kokkos_backend) + self.kokkos_arch = options.get("kokkos_arch", self.kokkos_arch) + self.adios2_mpi = options.get("adios2_mpi", self.adios2_mpi) + self.module_loads = data.get("module_loads", self.module_loads) + + def apps_summary(self) -> str: + chosen = [k for k, v in self.apps.items() if v] + return ", ".join(chosen) if chosen else "(none)" + + def to_json(self) -> str: + return json.dumps( + { + "cluster": self.cluster, + "write_modulefiles": self.write_modulefiles, + "overwrite": self.overwrite, + "install_prefix": self.install_prefix, + "dependencies": self.apps, + "versions": { + "Kokkos": self.kokkos_version, + "adios2": self.adios2_version, + }, + "options": { + "kokkos_backend": self.kokkos_backend, + "kokkos_arch": self.kokkos_arch, + "adios2_mpi": self.adios2_mpi, + }, + "module_loads": self.module_loads, + }, + indent=2, + ) + + +def unindent(script: str) -> str: + script_lines = script.splitlines() + min_indent = min( + (len(line) - len(line.lstrip()) for line in script_lines if line.strip()), + default=0, + ) + trimmed_lines = [line[min_indent:] for line in script_lines] + if trimmed_lines[0] == "": + trimmed_lines = trimmed_lines[1:] + if trimmed_lines[-1] == "": + trimmed_lines = trimmed_lines[:-1] + return "\n".join(trimmed_lines) + + +def InstallKokkosScriptModfile(settings: Settings) -> tuple[str, str]: + if settings.apps.get("Kokkos", False): + prefix = settings.install_prefix + version = settings.kokkos_version + backend = settings.kokkos_backend + arch = settings.kokkos_arch.strip() + modules = "\n".join( + [f"module load {module} && \\" for module in settings.module_loads] + ) + modules_in_module = "\n".join( + [f"module load {module}" for module in settings.module_loads] + ) + src_path = os.path.join(prefix, "src", "kokkos") + install_path = os.path.join(prefix, "kokkos", version, backend, arch.lower() if arch else "") + if os.path.exists(install_path) and not settings.overwrite: + raise FileExistsError( + f"Kokkos install path {install_path} already exists and overwrite is disabled" + ) + + extra_flags = " ".join(["-D " + kf for kf in settings.extra_kokkos_flags]) + cxx_standard = 20 if tuple(map(int, version.split("."))) >= (5, 0, 0) else 17 + + if arch == "": + arch = "NATIVE" + arch = arch.upper() + + script = f""" +# Kokkos installation +{modules} +rm -rf {src_path} && \\ +git clone https://github.com/kokkos/kokkos.git {src_path} && \\ +cd {src_path} && \\ +git checkout {version} && \\ +cmake -B build \\ + -D CMAKE_CXX_STANDARD={cxx_standard} \\ + -D CMAKE_CXX_EXTENSIONS=OFF \\ + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \\ + -D Kokkos_ARCH_{arch}=ON {f'-D Kokkos_ENABLE_{backend.upper()}=ON' if backend != 'cpu' else ''} \\ + -D CMAKE_INSTALL_PREFIX={install_path} {extra_flags} && \\ +cmake --build build -j $(nproc) && \\ +cmake --install build""" + + modfile = f""" +#%Module1.0###################################################################### +## +## Kokkos @ {backend} @ {arch} modulefile +## +################################################################################# +proc ModulesHelp {{ }} {{ + puts stderr \"\\tKokkos @ {backend} @ {arch}\\n\" +}} + +module-whatis \"Sets up Kokkos @ {backend} @ {arch}\" + +conflict kokkos +{modules_in_module} + +set basedir {install_path} +prepend-path PATH $basedir/bin +setenv Kokkos_DIR $basedir + +setenv Kokkos_ARCH_{arch} ON +{f'setenv Kokkos_ENABLE_{backend.upper()} ON' if backend != 'cpu' else ''}""" + + return (unindent(script), unindent(modfile)) + + else: + return ("""# skipping Kokkos install""", "") + + +def InstallAdios2Script(settings: Settings) -> tuple[str, str]: + if settings.apps.get("adios2", False): + prefix = settings.install_prefix + version = settings.adios2_version + mpi_mode = settings.adios2_mpi + modules = "\n".join( + [f"module load {module} && \\" for module in settings.module_loads] + ) + modules_in_module = "\n".join( + [f"module load {module}" for module in settings.module_loads] + ) + src_path = os.path.join(prefix, "src", "adios2") + install_path = os.path.join(prefix, "adios2", version, mpi_mode) + if os.path.exists(install_path) and not settings.overwrite: + raise FileExistsError( + f"Adios2 install path {install_path} already exists and overwrite is disabled" + ) + + extra_flags = " ".join(["-D " + af for af in settings.extra_adios2_flags]) + cxx_standard = ( + 20 + if tuple(map(int, settings.kokkos_version.split("."))) >= (5, 0, 0) + else 17 + ) + + with_mpi = "ON" if mpi_mode == "mpi" else "OFF" + + script = f""" +# Adios2 installation +{modules} +rm -rf {src_path} && \\ +git clone https://github.com/ornladios/ADIOS2.git {src_path} && \\ +cd {src_path} && \\ +git checkout v{version} && \\ +cmake -B build \\ + -D CMAKE_CXX_STANDARD={cxx_standard} \\ + -D CMAKE_CXX_EXTENSIONS=OFF \\ + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \\ + -D BUILD_SHARED_LIBS=ON \\ + -D ADIOS2_USE_Python=OFF \\ + -D ADIOS2_USE_Fortran=OFF \\ + -D ADIOS2_USE_ZeroMQ=OFF \\ + -D BUILD_TESTING=OFF \\ + -D ADIOS2_BUILD_EXAMPLES=OFF \\ + -D ADIOS2_USE_HDF5=OFF \\ + -D ADIOS2_USE_MPI={with_mpi} \\ + -D CMAKE_INSTALL_PREFIX={install_path} {extra_flags} && \\ +cmake --build build -j $(nproc) && \\ +cmake --install build""" + + modfile = f""" +#%Module1.0###################################################################### +## +## ADIOS2 @ {mpi_mode} modulefile +## +################################################################################# +proc ModulesHelp {{ }} {{ + puts stderr \"\\tADIOS2 @ {mpi_mode}\\n\" +}} + +module-whatis \"Sets up ADIOS2 @ {mpi_mode}\" + +conflict adios2 +{modules_in_module} + +set basedir {install_path} +prepend-path PATH $basedir/bin +setenv ADIOS2_DIR $basedir + +setenv ADIOS2_USE_MPI {with_mpi}""" + + return (unindent(script), unindent(modfile)) + + else: + return ("""# skipping Adios2 install""", "") + + +def InstallNt2pyScript(settings: Settings) -> str: + if settings.apps.get("nt2py", False): + prefix = settings.install_prefix + modules = "\n".join( + [f"module load {module} && \\" for module in settings.module_loads] + ) + install_path = f"{prefix}/.venv" + + script = f""" + # nt2py installation + {modules} + rm -rf {install_path} && \\ + python3 -m venv {install_path} && \\ + source {install_path}/bin/activate && \\ + pip install nt2py && \\ + deactivate + """ + return unindent(script) + else: + return """# skipping nt2py install""" + + +PRESETS = { + "rusty": { + "module_loads": ["openmpi/5.0.6.lua", "cuda/12.8.0.lua", "gcc/14.2.0.lua"], + "kokkos_backend": "cuda", + "kokkos_arch": "AMPERE80", + "adios2_mpi": "mpi", + }, + "stellar": {"module_loads": []}, + "perlmutter": { + "module_loads": ["gpu/1.0"], + "kokkos_backend": "cuda", + "kokkos_arch": "AMPERE80", + "extra_kokkos_flags": [ + "Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF", + "CMAKE_CXX_COMPILER=CC", + ], + "extra_adios2_flags": [ + "LIBFABRIC_ROOT=/opt/cray/libfabric/1.15.2.0/", + "MPI_ROOT=/opt/cray/pe/craype/2.7.30", + ], + }, + "lumi": { + "module_loads": ["PrgEnv-cray", "cray-mpich", "craype-accel-amd-gfx90a", "rocm"], + "kokkos_backend": "hip", + "kokkos_arch": "AMD_GFX90A", + "extra_kokkos_flags": [ + "CMAKE_CXX_COMPILER=hipcc", + "AMDGPU_TARGETS=gfx90a", + ], + "extra_adios2_flags": [ + "CMAKE_CXX_COMPILER=CC", + "CMAKE_C_COMPILER=cc" + ] + }, + "frontier": {"module_loads": []}, + "aurora": {"module_loads": []}, +} + + +def apply_preset(s: Settings, name: str) -> None: + s.cluster = name + s.install_prefix = os.path.join(os.path.expanduser("~"), ".entity") + cluster_preset = PRESETS.get(name, {}) + s.apps["Kokkos"] = True + s.apps["adios2"] = True + s.apps["nt2py"] = False + s.write_modulefiles = True + s.overwrite = True + s.module_loads = cluster_preset.get("module_loads", []) + s.kokkos_backend = cluster_preset.get("kokkos_backend", "cpu") + s.kokkos_arch = cluster_preset.get("kokkos_arch", "NATIVE") + s.adios2_mpi = cluster_preset.get("adios2_mpi", "mpi") + s.extra_kokkos_flags = cluster_preset.get("extra_kokkos_flags", []) + s.extra_adios2_flags = cluster_preset.get("extra_adios2_flags", []) + + +def on_install_confirmed(settings: Settings) -> None: + global MESSAGE + os.makedirs(settings.install_prefix, exist_ok=True) + kokkos_script, kokkos_modfile = InstallKokkosScriptModfile(settings) + adios2_script, adios2_modfile = InstallAdios2Script(settings) + with open(os.path.join(settings.install_prefix, "install.sh"), "w") as f: + f.write("#!/usr/bin/env bash\n\n") + f.write(kokkos_script) + f.write("\n\n") + f.write(adios2_script) + f.write("\n") + if settings.write_modulefiles: + os.makedirs(os.path.join(settings.install_prefix, "modules"), exist_ok=True) + if kokkos_modfile != "": + kokkos_modfile_file = os.path.join( + settings.install_prefix, + "modules", + "kokkos", + settings.kokkos_version, + settings.kokkos_backend, + settings.kokkos_arch.strip().lower(), + ) + os.makedirs(os.path.dirname(kokkos_modfile_file), exist_ok=True) + if os.path.exists(kokkos_modfile_file) and not settings.overwrite: + raise FileExistsError( + f"modulefile {kokkos_modfile_file} already exists and overwrite is disabled" + ) + with open(kokkos_modfile_file, "w") as f: + f.write(kokkos_modfile) + if adios2_modfile != "": + adios2_modfile_file = os.path.join( + settings.install_prefix, + "modules", + "adios2", + settings.adios2_version, + settings.adios2_mpi, + ) + os.makedirs(os.path.dirname(adios2_modfile_file), exist_ok=True) + if os.path.exists(adios2_modfile_file) and not settings.overwrite: + raise FileExistsError( + f"modulefile {adios2_modfile_file} already exists and overwrite is disabled" + ) + with open(adios2_modfile_file, "w") as f: + f.write(adios2_modfile) + + os.chmod(os.path.join(settings.install_prefix, "install.sh"), 0o755) + MESSAGE = f"- installation script written to {os.path.join(settings.install_prefix, 'install.sh')}!\n" + MESSAGE += " please read and verify it before running.\n\n" + if settings.write_modulefiles: + MESSAGE += f"- module files have been written to {os.path.join(settings.install_prefix, 'modules')} directory.\n" + MESSAGE += f" add them to your .rc script as `module use --append {os.path.join(settings.install_prefix, 'modules')}`\n\n" + + if settings.apps.get("nt2py", False): + MESSAGE += ( + "- nt2py installed in a new virtual environment at " + f"{os.path.join(settings.install_prefix, '.venv')}.\n" + ) + MESSAGE += " activate it with `source {}/bin/activate`.\n\n".format( + os.path.join(settings.install_prefix, ".venv") + ) + + settings_json = os.path.join(settings.install_prefix, "settings.json") + with open(settings_json, "w") as f: + f.write(settings.to_json()) + return + + +@dataclass +class MenuItem: + label: str + hint: str = "" + right: Optional[Callable[[], str]] = None + on_enter: Optional[Callable[[], None]] = None + on_space: Optional[Callable[[], None]] = None + disabled: Optional[Callable[[], bool]] = None + + +class TuiExitInstall(Exception): + pass + + +class App: + def __init__(self, stdscr): + self.stdscr = stdscr + self.s = Settings() + if os.path.exists(os.path.join(self.s.install_prefix, "settings.json")): + with open(os.path.join(self.s.install_prefix, "settings.json"), "r") as f: + data = json.load(f) + self.s.from_json(json.dumps(data)) + + self.state = "mainmenu" + self.stack: List[Tuple[str, int]] = [] + self.selected = 0 + self.scroll = 0 + self.message = "use arrows or j/k" + + self.mod_sel = 0 + self.mod_scroll = 0 + + self._init_curses() + + def _init_curses(self) -> None: + curses.curs_set(0) + self.stdscr.keypad(True) + curses.noecho() + curses.cbreak() + + if curses.has_colors(): + curses.start_color() + curses.use_default_colors() + curses.init_pair(PAIR_TITLE, COLOR_TITLE_FG, -1) + curses.init_pair(PAIR_TEXT, COLOR_TEXT_FG, -1) + curses.init_pair(PAIR_SELECTED, COLOR_SELECTED_FG, COLOR_SELECTED_BG) + curses.init_pair(PAIR_HINT, COLOR_HINT_FG, -1) + curses.init_pair(PAIR_OK, COLOR_OK_FG, -1) + curses.init_pair(PAIR_ERR, COLOR_ERR_FG, -1) + curses.init_pair(PAIR_KEY, COLOR_KEY_FG, -1) + curses.init_pair(PAIR_DIM, COLOR_DIM_FG, -1) + + def cp(self, pair_id: int) -> int: + return curses.color_pair(pair_id) if curses.has_colors() else 0 + + # ----- formatting helpers ----- + + def checkbox(self, on: bool) -> str: + return "[x]" if on else "[ ]" + + def pill(self, on: bool) -> str: + return "[on]" if on else "[off]" + + def kokkos_right(self) -> str: + arch = self.s.kokkos_arch.strip() or "-" + return f"{self.s.kokkos_version} · {self.s.kokkos_backend} · {arch}" + + def adios2_right(self) -> str: + return f"{self.s.adios2_version} · {self.s.adios2_mpi}" + + # ----- nav stack ----- + + def push(self, st: str) -> None: + self.stack.append((self.state, self.selected)) + self.state = st + self.selected = 0 + self.scroll = 0 + self.message = "" + + def pop(self) -> None: + if self.stack: + self.state, self.selected = self.stack.pop() + else: + self.state, self.selected = "mainmenu", 0 + self.scroll = 0 + self.message = "" + + # ----- drawing ----- + + def add(self, y: int, x: int, s: str, attr: int = 0) -> None: + try: + self.stdscr.addstr(y, x, s, attr) + except curses.error: + pass + + def hline(self, y: int) -> None: + _, w = self.stdscr.getmaxyx() + try: + self.stdscr.hline(y, 0, curses.ACS_HLINE, max(0, w - 1)) + except curses.error: + pass + + def draw_keybar(self, y: int, x: int, pairs: List[Tuple[str, str]]) -> None: + cur_x = x + for key, action in pairs: + self.add(y, cur_x, key, self.cp(PAIR_KEY) | curses.A_BOLD) + cur_x += len(key) + self.add(y, cur_x, " ", self.cp(PAIR_DIM)) + cur_x += 1 + self.add(y, cur_x, action, self.cp(PAIR_HINT)) + cur_x += len(action) + self.add(y, cur_x, " ", self.cp(PAIR_DIM)) + cur_x += 3 + + def breadcrumb(self) -> str: + if self.state == "mainmenu": + return "mainmenu" + if self.state == "custom": + return "mainmenu › custom install" + if self.state == "dependencies": + return "mainmenu › custom install › dependencies" + if self.state == "versions": + return "mainmenu › custom install › versions" + if self.state == "options": + return "mainmenu › custom install › options" + if self.state == "cluster": + return "mainmenu › cluster-specific" + if self.state == "preset_applied": + return f"mainmenu › cluster-specific › {self.s.cluster}" + return "mainmenu" + + def draw_menu(self, title: str, prompt: str, items: List[MenuItem]) -> None: + self.stdscr.erase() + h, w = self.stdscr.getmaxyx() + + self.add(0, 2, title, self.cp(PAIR_TITLE) | curses.A_BOLD) + bc = self.breadcrumb() + self.add(0, max(2, w - 2 - len(bc)), bc, self.cp(PAIR_DIM)) + + self.draw_keybar( + 1, + 2, + [ + ("↑/↓/j/k", "move"), + ("enter", "select"), + ("space", "toggle/cycle"), + ("b", "back"), + ("q", "quit"), + ], + ) + self.hline(2) + + status1 = f"cluster: {self.s.cluster} write modulefiles: {self.pill(self.s.write_modulefiles)} module loads: {len(self.s.module_loads)}" + status2 = ( + f"prefix: {self.s.install_prefix} dependencies: {self.s.apps_summary()}" + ) + self.add(3, 2, status1[: w - 4], self.cp(PAIR_TEXT)) + self.add(4, 2, status2[: w - 4], self.cp(PAIR_TEXT)) + self.hline(5) + + self.add(6, 2, prompt[: w - 4], self.cp(PAIR_TEXT) | curses.A_BOLD) + + list_y = 8 + footer_h = 3 + view_h = max(1, h - list_y - footer_h) + n = len(items) + + if n == 0: + self.add(list_y, 2, "(empty)", self.cp(PAIR_HINT)) + else: + self.selected = max(0, min(self.selected, n - 1)) + + if self.selected < self.scroll: + self.scroll = self.selected + if self.selected >= self.scroll + view_h: + self.scroll = self.selected - view_h + 1 + self.scroll = max(0, min(self.scroll, max(0, n - view_h))) + + shown = items[self.scroll : self.scroll + view_h] + + for i, it in enumerate(shown): + idx = self.scroll + i + sel = idx == self.selected + dis = bool(it.disabled and it.disabled()) + + row_attr = ( + self.cp(PAIR_SELECTED) | curses.A_BOLD + if sel + else (self.cp(PAIR_DIM) if dis else self.cp(PAIR_TEXT)) + ) + self.add(list_y + i, 2, f" {it.label}"[: w - 4], row_attr) + + if it.right: + rt = (it.right() or "").strip() + if rt: + rt = rt[: max(0, w - 6)] + x = max(2, w - 2 - len(rt)) + rt_attr = ( + row_attr + if sel + else (self.cp(PAIR_HINT) if not dis else self.cp(PAIR_DIM)) + ) + self.add(list_y + i, x, rt, rt_attr) + + if sel and it.hint: + self.add( + list_y + i, + min(w - 4, 30), + f" {it.hint}"[: w - 4], + self.cp(PAIR_HINT), + ) + + self.hline(h - 3) + msg = self.message or "" + if msg: + is_err = msg.startswith("error") + attr = (self.cp(PAIR_ERR) if is_err else self.cp(PAIR_OK)) | curses.A_BOLD + self.add(h - 2, 2, msg[: w - 4], attr) + self.stdscr.refresh() + + # ----- modals ----- + + def input_box(self, title: str, prompt: str, initial: str) -> Optional[str]: + h, w = self.stdscr.getmaxyx() + win_h, win_w = 9, min(86, max(46, w - 6)) + top, left = max(0, (h - win_h) // 2), max(0, (w - win_w) // 2) + + win = curses.newwin(win_h, win_w, top, left) + win.keypad(True) + win.border() + + win.addstr(1, 2, title[: win_w - 4], self.cp(PAIR_TITLE) | curses.A_BOLD) + win.addstr(2, 2, prompt[: win_w - 4], self.cp(PAIR_TEXT)) + + buf = list(initial) + curses.curs_set(1) + + while True: + win.addstr(4, 2, " " * (win_w - 4), self.cp(PAIR_TEXT)) + text = "".join(buf) + if len(text) > win_w - 4: + text = text[-(win_w - 4) :] + win.addstr(4, 2, text, self.cp(PAIR_TEXT) | curses.A_BOLD) + win.addstr(6, 2, "enter=ok esc=cancel", self.cp(PAIR_DIM)) + win.refresh() + + ch = win.getch() + if ch == 27: + curses.curs_set(0) + return None + if ch in (curses.KEY_ENTER, 10, 13): + curses.curs_set(0) + return "".join(buf).strip() + if ch in (curses.KEY_BACKSPACE, 127, 8): + if buf: + buf.pop() + elif 32 <= ch <= 126: + buf.append(chr(ch)) + + def confirm_install(self) -> bool: + arch = self.s.kokkos_arch.strip() or "-" + lines = [ + f"cluster: {self.s.cluster}", + f"overwrite existing files: {self.pill(self.s.overwrite)}", + f"write modulefiles: {self.pill(self.s.write_modulefiles)}", + f"module loads: {len(self.s.module_loads)}", + f"prefix: {self.s.install_prefix}", + f"dependencies: {self.s.apps_summary()}", + f"kokkos: {self.s.kokkos_version} · {self.s.kokkos_backend} · {arch}", + f"adios2: {self.s.adios2_version} · {self.s.adios2_mpi}", + "", + "confirm install?", + ] + + h, w = self.stdscr.getmaxyx() + win_h, win_w = min(16, max(10, h - 6)), min(94, max(52, w - 6)) + top, left = max(0, (h - win_h) // 2), max(0, (w - win_w) // 2) + + win = curses.newwin(win_h, win_w, top, left) + win.keypad(True) + win.border() + win.addstr(1, 2, "confirm", self.cp(PAIR_TITLE) | curses.A_BOLD) + + y = 3 + for ln in lines[: win_h - 6]: + win.addstr(y, 2, ln[: win_w - 4], self.cp(PAIR_TEXT)) + y += 1 + + win.addstr(win_h - 3, 2, "y=yes n=no", self.cp(PAIR_DIM)) + win.refresh() + + while True: + ch = win.getch() + if ch in (ord("y"), ord("Y")): + return True + if ch in (ord("n"), ord("N"), 27): + return False + + # ----- helpers ----- + + def cycle(self, current: str, options: List[str]) -> str: + if current not in options: + return options[0] + i = options.index(current) + return options[(i + 1) % len(options)] + + # ----- module editor ----- + + def module_editor(self) -> None: + while True: + self.stdscr.erase() + h, w = self.stdscr.getmaxyx() + + self.add(0, 2, "module lines", self.cp(PAIR_TITLE) | curses.A_BOLD) + self.draw_keybar( + 1, + 2, + [ + ("↑/↓/j/k", "move"), + ("enter", "edit"), + ("a", "add"), + ("d", "delete"), + ("u/m", "reorder"), + ("b", "back"), + ], + ) + self.hline(2) + + self.add(3, 2, f"lines: {len(self.s.module_loads)}", self.cp(PAIR_TEXT)) + self.hline(4) + + lines = self.s.module_loads + n = len(lines) + list_y = 6 + view_h = max(1, h - list_y - 3) + + if n == 0: + self.add(list_y, 2, "(empty) press a to add", self.cp(PAIR_HINT)) + else: + self.mod_sel = max(0, min(self.mod_sel, n - 1)) + if self.mod_sel < self.mod_scroll: + self.mod_scroll = self.mod_sel + if self.mod_sel >= self.mod_scroll + view_h: + self.mod_scroll = self.mod_sel - view_h + 1 + self.mod_scroll = max(0, min(self.mod_scroll, max(0, n - view_h))) + + shown = lines[self.mod_scroll : self.mod_scroll + view_h] + for i, ln in enumerate(shown): + idx = self.mod_scroll + i + sel = idx == self.mod_sel + attr = ( + self.cp(PAIR_SELECTED) | curses.A_BOLD + if sel + else self.cp(PAIR_TEXT) + ) + self.add(list_y + i, 2, f" {ln}"[: w - 4], attr) + + self.hline(h - 3) + self.add( + h - 2, + 2, + "tip: example: cuda/12.9"[: w - 4], + self.cp(PAIR_HINT), + ) + self.stdscr.refresh() + + ch = self.stdscr.getch() + if ch in (ord("q"), ord("Q"), ord("b"), 8, 127): + return + + n = len(self.s.module_loads) + self.mod_sel = 0 if n == 0 else max(0, min(self.mod_sel, n - 1)) + + if ch in (curses.KEY_UP, ord("k"), ord("K")) and n: + self.mod_sel = (self.mod_sel - 1) % n + continue + if ch in (curses.KEY_DOWN, ord("j"), ord("J")) and n: + self.mod_sel = (self.mod_sel + 1) % n + continue + + if ch in (ord("a"), ord("A")): + val = self.input_box("add module line", "example: cuda/12.9", "") + if val: + self.s.module_loads.append(val) + self.mod_sel = len(self.s.module_loads) - 1 + continue + + if ch in (ord("d"), ord("D")): + if n == 0: + continue + val = self.input_box("delete line", "type 'delete' to confirm:", "") + if val == "delete": + del self.s.module_loads[self.mod_sel] + self.mod_sel = max( + 0, min(self.mod_sel, len(self.s.module_loads) - 1) + ) + continue + + if ch in (ord("u"), ord("U")): + if n >= 2 and self.mod_sel > 0: + i = self.mod_sel + self.s.module_loads[i - 1], self.s.module_loads[i] = ( + self.s.module_loads[i], + self.s.module_loads[i - 1], + ) + self.mod_sel -= 1 + continue + + if ch in (ord("m"), ord("M")): + if n >= 2 and self.mod_sel < n - 1: + i = self.mod_sel + self.s.module_loads[i + 1], self.s.module_loads[i] = ( + self.s.module_loads[i], + self.s.module_loads[i + 1], + ) + self.mod_sel += 1 + continue + + if ch in (curses.KEY_ENTER, 10, 13): + if n == 0: + continue + cur = self.s.module_loads[self.mod_sel] + val = self.input_box("edit module line", "edit the selected line:", cur) + if val is not None and val.strip(): + self.s.module_loads[self.mod_sel] = val.strip() + continue + + # ----- menus ----- + + def versions_menu(self) -> Tuple[str, str, List[MenuItem]]: + def edit_kokkos(): + val = self.input_box( + "kokkos version", "enter version/tag:", self.s.kokkos_version + ) + if val: + self.s.kokkos_version = val.strip() + + def edit_adios2(): + val = self.input_box( + "adios2 version", "enter version/tag:", self.s.adios2_version + ) + if val: + self.s.adios2_version = val.strip() + + return ( + "versions", + "set versions:", + [ + MenuItem( + "kokkos version", + "enter to edit", + right=lambda: self.s.kokkos_version, + on_enter=edit_kokkos, + ), + MenuItem( + "adios2 version", + "enter to edit", + right=lambda: self.s.adios2_version, + on_enter=edit_adios2, + ), + MenuItem("back", "return", on_enter=self.pop), + ], + ) + + def options_menu(self) -> Tuple[str, str, List[MenuItem]]: + def cycle_kokkos(): + self.s.kokkos_backend = self.cycle(self.s.kokkos_backend, KOKKOS_BACKENDS) + + def edit_kokkos_arch(): + val = self.input_box( + "kokkos arch", "enter arch text (free-form):", self.s.kokkos_arch + ) + if val is not None: + self.s.kokkos_arch = val.strip() + + def cycle_adios2(): + self.s.adios2_mpi = self.cycle(self.s.adios2_mpi, ADIOS2_MPI_MODES) + + return ( + "options", + "set build options:", + [ + MenuItem( + "kokkos backend", + "space cycles: cpu/cuda/hip/sycl", + right=lambda: self.s.kokkos_backend, + on_enter=cycle_kokkos, + on_space=cycle_kokkos, + disabled=lambda: not self.s.apps.get("Kokkos", False), + ), + MenuItem( + "kokkos arch", + "enter to edit (optional)", + right=lambda: (self.s.kokkos_arch.strip() or "-"), + on_enter=edit_kokkos_arch, + disabled=lambda: not self.s.apps.get("Kokkos", False), + ), + MenuItem( + "adios2 mpi", + "space cycles: non-mpi/mpi", + right=lambda: self.s.adios2_mpi, + on_enter=cycle_adios2, + on_space=cycle_adios2, + disabled=lambda: not self.s.apps.get("adios2", False), + ), + MenuItem("back", "return", on_enter=self.pop), + ], + ) + + def menu_main(self) -> Tuple[str, str, List[MenuItem]]: + return ( + "entity deps", + "main menu:", + [ + MenuItem( + "custom install", + "edit settings then install", + on_enter=lambda: self.push("custom"), + ), + MenuItem( + "cluster-specific", + "apply a cluster-specific preset (editable)", + on_enter=lambda: self.push("cluster"), + ), + MenuItem("exit", "", on_enter=lambda: setattr(self, "state", "exit")), + ], + ) + + def menu_custom(self) -> Tuple[str, str, List[MenuItem]]: + def toggle_write_modulefiles(): + self.s.write_modulefiles = not self.s.write_modulefiles + + def toggle_overwrite(): + self.s.overwrite = not self.s.overwrite + + def edit_prefix(): + val = self.input_box( + "install location", "enter install prefix:", self.s.install_prefix + ) + if val: + self.s.install_prefix = os.path.expanduser(val.strip()) + + def go_apps(): + self.push("dependencies") + + def go_versions(): + self.push("versions") + + def go_options(): + self.push("options") + + def do_install(): + if not self.confirm_install(): + self.message = "cancelled." + return + on_install_confirmed(self.s) + raise TuiExitInstall + + return ( + "custom install", + "settings:", + [ + MenuItem( + "overwrite existing files", + "whether to overwrite existing files", + right=lambda: "enabled" if self.s.overwrite else "disabled", + on_enter=toggle_overwrite, + on_space=toggle_overwrite, + ), + MenuItem( + "write modulefiles", + "whether to create module files", + right=lambda: "enabled" if self.s.write_modulefiles else "disabled", + on_enter=toggle_write_modulefiles, + on_space=toggle_write_modulefiles, + ), + MenuItem( + "module load lines", + "add/remove modules to load", + right=lambda: f"{len(self.s.module_loads)} entry(s)", + on_enter=self.module_editor, + ), + MenuItem( + "install location", + "root location where modules and dependencies are installed", + right=lambda: self.s.install_prefix, + on_enter=edit_prefix, + ), + MenuItem( + "dependencies to install", + "select which dependencies to install", + right=lambda: self.s.apps_summary(), + on_enter=go_apps, + ), + MenuItem( + "versions", + "edit dependency versions", + right=lambda: " · ".join( + [ + a + for (a, ae) in zip( + [ + f"kokkos {self.s.kokkos_version}", + f"adios2 {self.s.adios2_version}", + ], + [ + self.s.apps.get(app, False) + for app in ["Kokkos", "adios2"] + ], + ) + if ae + ] + ), + on_enter=go_versions, + ), + MenuItem( + "options", + "pick backends/architectures/mpi", + right=lambda: " · ".join( + [ + a + for (a, ae) in zip( + [ + f"kokkos {self.s.kokkos_backend}/{self.s.kokkos_arch.strip() or '-'}", + f"adios2 {self.s.adios2_mpi}", + ], + [ + self.s.apps.get(app, False) + for app in ["Kokkos", "adios2"] + ], + ) + if ae + ] + ), + on_enter=go_options, + ), + MenuItem("install", "", on_enter=do_install), + MenuItem("back", "", on_enter=self.pop), + ], + ) + + def menu_apps(self) -> Tuple[str, str, List[MenuItem]]: + def toggle(k: str): + self.s.apps[k] = not self.s.apps.get(k, False) + + return ( + "dependencies", + "select the dependencies:", + [ + MenuItem( + f"{self.checkbox(self.s.apps.get('Kokkos', False))} kokkos", + "", + on_enter=lambda: toggle("Kokkos"), + on_space=lambda: toggle("Kokkos"), + right=self.kokkos_right, + ), + MenuItem( + f"{self.checkbox(self.s.apps.get('adios2', False))} adios2", + "", + on_enter=lambda: toggle("adios2"), + on_space=lambda: toggle("adios2"), + right=self.adios2_right, + ), + MenuItem( + f"{self.checkbox(self.s.apps.get('nt2py', False))} nt2py", + "", + on_enter=lambda: toggle("nt2py"), + on_space=lambda: toggle("nt2py"), + ), + MenuItem("back", "", on_enter=self.pop), + ], + ) + + def menu_cluster(self) -> Tuple[str, str, List[MenuItem]]: + def choose(name: str): + print ("CALLING:", name) + apply_preset(self.s, name) + self.push("custom") + + return ( + "cluster-specific", + "pick a preset:", + [MenuItem(cluster, "apply preset", on_enter=lambda c=cluster: choose(c)) for cluster in list(PRESETS.keys())] + [MenuItem("back", "", on_enter=self.pop)], + ) + + def get_menu(self) -> Tuple[str, str, List[MenuItem]]: + if self.state == "mainmenu": + return self.menu_main() + if self.state == "custom": + return self.menu_custom() + if self.state == "dependencies": + return self.menu_apps() + if self.state == "versions": + return self.versions_menu() + if self.state == "options": + return self.options_menu() + if self.state == "cluster": + return self.menu_cluster() + self.state = "mainmenu" + return self.menu_main() + + # ----- navigation ----- + + def is_disabled(self, it: MenuItem) -> bool: + return bool(it.disabled and it.disabled()) + + def move_sel(self, items: List[MenuItem], delta: int) -> None: + if not items: + return + n = len(items) + start = self.selected + for _ in range(n): + self.selected = (self.selected + delta) % n + if not self.is_disabled(items[self.selected]): + return + self.selected = start + + def activate(self, items: List[MenuItem], enter: bool) -> None: + if not items: + return + it = items[self.selected] + if self.is_disabled(it): + self.message = "error: option disabled." + return + fn = it.on_enter if enter else it.on_space + if fn: + fn() + + # ----- loop ----- + + def run(self) -> None: + while True: + if self.state == "exit": + return + + title, prompt, items = self.get_menu() + self.draw_menu(title, prompt, items) + + ch = self.stdscr.getch() + + if ch in (ord("q"), ord("Q")): + self.state = "exit" + continue + + if ch in (ord("b"), 8, 127): + self.pop() + continue + + if ch in (curses.KEY_UP, ord("k"), ord("K")): + self.move_sel(items, -1) + continue + + if ch in (curses.KEY_DOWN, ord("j"), ord("J")): + self.move_sel(items, +1) + continue + + if ch in (curses.KEY_ENTER, 10, 13): + self.activate(items, enter=True) + continue + + if ch == ord(" "): + self.activate(items, enter=False) + continue + + +def _wrapper_capture(stdscr) -> None: + app = App(stdscr) + try: + app.run() + except TuiExitInstall: + raise + + +if __name__ == "__main__": + try: + curses.wrapper(_wrapper_capture) + raise SystemExit(0) + except TuiExitInstall: + print(MESSAGE) + raise SystemExit(0) + except KeyboardInterrupt: + raise SystemExit(130) diff --git a/dev/nix/adios2.nix b/dev/nix/adios2.nix index a3890b788..810c228dc 100644 --- a/dev/nix/adios2.nix +++ b/dev/nix/adios2.nix @@ -6,9 +6,9 @@ let name = "adios2"; - version = "2.10.2"; + version = "2.11.0"; cmakeFlags = { - CMAKE_CXX_STANDARD = "17"; + CMAKE_CXX_STANDARD = "20"; CMAKE_CXX_EXTENSIONS = "OFF"; CMAKE_POSITION_INDEPENDENT_CODE = "TRUE"; BUILD_SHARED_LIBS = "ON"; @@ -30,7 +30,7 @@ stdenv.mkDerivation { src = pkgs.fetchgit { url = "https://github.com/ornladios/ADIOS2/"; rev = "v${version}"; - sha256 = "sha256-NVyw7xoPutXeUS87jjVv1YxJnwNGZAT4QfkBLzvQbwg="; + sha256 = "sha256-yHPI///17poiCEb7Luu5qfqxTWm9Nh+o9r57mZT26U0="; }; nativeBuildInputs = with pkgs; [ diff --git a/dev/nix/kokkos.nix b/dev/nix/kokkos.nix index 7d86e665b..f51d2c685 100644 --- a/dev/nix/kokkos.nix +++ b/dev/nix/kokkos.nix @@ -7,25 +7,26 @@ let name = "kokkos"; - pversion = "4.7.01"; + pversion = "5.0.1"; compilerPkgs = { "HIP" = with pkgs.rocmPackages; [ - llvm.rocm-merged-llvm + clang rocm-core clr rocthrust rocprim rocminfo rocm-smi + pkgs.clang-tools ]; "CUDA" = with pkgs.cudaPackages; [ - llvmPackages_18.clang-tools + pkgs.clang-tools cudatoolkit cuda_cudart pkgs.gcc13 ]; "NONE" = [ - pkgs.llvmPackages_18.clang-tools + pkgs.clang-tools pkgs.gcc13 ]; }; @@ -39,8 +40,8 @@ let "HIP" = [ "-D Kokkos_ENABLE_HIP=ON" "-D Kokkos_ARCH_${getArch { }}=ON" - "-D AMDGPU_TARGETS=${builtins.replaceStrings [ "amd_" ] [ "" ] (pkgs.lib.toLower (getArch { }))}" - "-D CMAKE_CXX_COMPILER=hipcc" + "-D GPU_TARGETS=${builtins.replaceStrings [ "amd_" ] [ "" ] (pkgs.lib.toLower (getArch { }))}" + "-D CMAKE_CXX_COMPILER=clang++" ]; "CUDA" = [ "-D Kokkos_ENABLE_CUDA=ON" @@ -56,7 +57,7 @@ pkgs.stdenv.mkDerivation rec { src = pkgs.fetchgit { url = "https://github.com/kokkos/kokkos/"; rev = "${pversion}"; - sha256 = "sha256-MgphOsKE8umgYxVQZzex+elgvDDC09JaMCoU5YXaLco="; + sha256 = "sha256-ChpwGBwE7sNovjdAM/iCeOqqwGufKxAh5vQ3qK6aFBU="; }; nativeBuildInputs = with pkgs; [ @@ -78,7 +79,7 @@ pkgs.stdenv.mkDerivation rec { configurePhase = '' cmake -B build -D CMAKE_BUILD_TYPE=Release \ - -D CMAKE_CXX_STANDARD=17 \ + -D CMAKE_CXX_STANDARD=20 \ -D CMAKE_CXX_EXTENSIONS=OFF \ -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \ ${pkgs.lib.concatStringsSep " " cmakeExtraFlags.${gpu}} \ diff --git a/dev/nix/shell.nix b/dev/nix/shell.nix index 0d4cc9119..c1a9dd165 100644 --- a/dev/nix/shell.nix +++ b/dev/nix/shell.nix @@ -29,8 +29,8 @@ let CC = "gcc"; }; HIP = { - CXX = "hipcc"; - CC = "hipcc"; + CXX = "clang++"; + CC = "clang"; }; CUDA = { }; }; @@ -45,8 +45,7 @@ pkgs.mkShell { adios2Pkg kokkosPkg - python312 - python312Packages.jupyter + python314 cmake-format cmake-lint diff --git a/extern/Kokkos b/extern/Kokkos index 1b1383c60..37f70304d 160000 --- a/extern/Kokkos +++ b/extern/Kokkos @@ -1 +1 @@ -Subproject commit 1b1383c6001f3bfe9fe309ca923c2d786600cc79 +Subproject commit 37f70304dc3676691af88d3ac3ba50cddbfa337f diff --git a/extern/adios2 b/extern/adios2 index a19dad6ce..1ef0b5797 160000 --- a/extern/adios2 +++ b/extern/adios2 @@ -1 +1 @@ -Subproject commit a19dad6cecb00319825f20fd9f455ebbab903d34 +Subproject commit 1ef0b5797aeb8a1cc1bb36ec4089eaec19a2eea0 diff --git a/extern/plog b/extern/plog deleted file mode 160000 index e21baecd4..000000000 --- a/extern/plog +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e21baecd4753f14da64ede979c5a19302618b752 diff --git a/include/plog/Appenders/AndroidAppender.h b/include/plog/Appenders/AndroidAppender.h new file mode 100644 index 000000000..722488ab2 --- /dev/null +++ b/include/plog/Appenders/AndroidAppender.h @@ -0,0 +1,47 @@ +#pragma once +#include +#include + +namespace plog +{ + template + class PLOG_LINKAGE_HIDDEN AndroidAppender : public IAppender + { + public: + AndroidAppender(const char* tag) : m_tag(tag) + { + } + + virtual void write(const Record& record) PLOG_OVERRIDE + { + std::string str = Formatter::format(record); + + __android_log_print(toPriority(record.getSeverity()), m_tag, "%s", str.c_str()); + } + + private: + static android_LogPriority toPriority(Severity severity) + { + switch (severity) + { + case fatal: + return ANDROID_LOG_FATAL; + case error: + return ANDROID_LOG_ERROR; + case warning: + return ANDROID_LOG_WARN; + case info: + return ANDROID_LOG_INFO; + case debug: + return ANDROID_LOG_DEBUG; + case verbose: + return ANDROID_LOG_VERBOSE; + default: + return ANDROID_LOG_UNKNOWN; + } + } + + private: + const char* const m_tag; + }; +} diff --git a/include/plog/Appenders/ArduinoAppender.h b/include/plog/Appenders/ArduinoAppender.h new file mode 100644 index 000000000..276af323f --- /dev/null +++ b/include/plog/Appenders/ArduinoAppender.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include + +namespace plog +{ + template + class PLOG_LINKAGE_HIDDEN ArduinoAppender : public IAppender + { + public: + ArduinoAppender(Stream &stream) : m_stream(stream) + { + } + + virtual void write(const Record &record) PLOG_OVERRIDE + { + m_stream.print(Formatter::format(record).c_str()); + } + + private: + Stream &m_stream; + }; +} diff --git a/include/plog/Appenders/ColorConsoleAppender.h b/include/plog/Appenders/ColorConsoleAppender.h new file mode 100644 index 000000000..24bbc7d90 --- /dev/null +++ b/include/plog/Appenders/ColorConsoleAppender.h @@ -0,0 +1,108 @@ +#pragma once +#include +#include + +namespace plog +{ + template + class PLOG_LINKAGE_HIDDEN ColorConsoleAppender : public ConsoleAppender + { + public: +#ifdef _WIN32 +# ifdef _MSC_VER +# pragma warning(suppress: 26812) // Prefer 'enum class' over 'enum' +# endif + ColorConsoleAppender(OutputStream outStream = streamStdOut) + : ConsoleAppender(outStream) + , m_originalAttr() + { + if (this->m_isatty) + { + CONSOLE_SCREEN_BUFFER_INFO csbiInfo; + GetConsoleScreenBufferInfo(this->m_outputHandle, &csbiInfo); + + m_originalAttr = csbiInfo.wAttributes; + } + } +#else + ColorConsoleAppender(OutputStream outStream = streamStdOut) + : ConsoleAppender(outStream) + {} +#endif + + virtual void write(const Record& record) PLOG_OVERRIDE + { + util::nstring str = Formatter::format(record); + util::MutexLock lock(this->m_mutex); + + setColor(record.getSeverity()); + this->writestr(str); + resetColor(); + } + + protected: + void setColor(Severity severity) + { + if (this->m_isatty) + { + switch (severity) + { +#ifdef _WIN32 + case fatal: + SetConsoleTextAttribute(this->m_outputHandle, foreground::kRed | foreground::kGreen | foreground::kBlue | foreground::kIntensity | background::kRed); // white on red background + break; + + case error: + SetConsoleTextAttribute(this->m_outputHandle, static_cast(foreground::kRed | foreground::kIntensity | (m_originalAttr & 0xf0))); // red + break; + + case warning: + SetConsoleTextAttribute(this->m_outputHandle, static_cast(foreground::kRed | foreground::kGreen | foreground::kIntensity | (m_originalAttr & 0xf0))); // yellow + break; + + case debug: + case verbose: + SetConsoleTextAttribute(this->m_outputHandle, static_cast(foreground::kGreen | foreground::kBlue | foreground::kIntensity | (m_originalAttr & 0xf0))); // cyan + break; +#else + case fatal: + this->m_outputStream << "\x1B[97m\x1B[41m"; // white on red background + break; + + case error: + this->m_outputStream << "\x1B[91m"; // red + break; + + case warning: + this->m_outputStream << "\x1B[93m"; // yellow + break; + + case debug: + case verbose: + this->m_outputStream << "\x1B[96m"; // cyan + break; +#endif + default: + break; + } + } + } + + void resetColor() + { + if (this->m_isatty) + { +#ifdef _WIN32 + SetConsoleTextAttribute(this->m_outputHandle, m_originalAttr); +#else + this->m_outputStream << "\x1B[0m\x1B[0K"; +#endif + } + } + + private: +#ifdef _WIN32 + WORD m_originalAttr; +#endif + }; +} diff --git a/include/plog/Appenders/ConsoleAppender.h b/include/plog/Appenders/ConsoleAppender.h new file mode 100644 index 000000000..a8925a049 --- /dev/null +++ b/include/plog/Appenders/ConsoleAppender.h @@ -0,0 +1,83 @@ +#pragma once +#include +#include +#include +#include + +namespace plog +{ + enum OutputStream + { + streamStdOut, + streamStdErr + }; + + template + class PLOG_LINKAGE_HIDDEN ConsoleAppender : public IAppender + { + public: +#ifdef _WIN32 +# ifdef _MSC_VER +# pragma warning(suppress: 26812) // Prefer 'enum class' over 'enum' +# endif + ConsoleAppender(OutputStream outStream = streamStdOut) + : m_isatty(!!_isatty(_fileno(outStream == streamStdOut ? stdout : stderr))) + , m_outputStream(outStream == streamStdOut ? std::cout : std::cerr) + , m_outputHandle() + { + if (m_isatty) + { + m_outputHandle = GetStdHandle(outStream == streamStdOut ? stdHandle::kOutput : stdHandle::kErrorOutput); + } + } +#else + ConsoleAppender(OutputStream outStream = streamStdOut) + : m_isatty(!!isatty(fileno(outStream == streamStdOut ? stdout : stderr))) + , m_outputStream(outStream == streamStdOut ? std::cout : std::cerr) + {} +#endif + + virtual void write(const Record& record) PLOG_OVERRIDE + { + util::nstring str = Formatter::format(record); + util::MutexLock lock(m_mutex); + + writestr(str); + } + + protected: + void writestr(const util::nstring& str) + { +#ifdef _WIN32 + if (m_isatty) + { + const std::wstring& wstr = util::toWide(str); + WriteConsoleW(m_outputHandle, wstr.c_str(), static_cast(wstr.size()), NULL, NULL); + } + else + { +# if PLOG_CHAR_IS_UTF8 + m_outputStream << str << std::flush; +# else + m_outputStream << util::toNarrow(str, codePage::kActive) << std::flush; +# endif + } +#else + m_outputStream << str << std::flush; +#endif + } + + private: +#ifdef __BORLANDC__ + static int _isatty(int fd) { return ::isatty(fd); } +#endif + + protected: + util::Mutex m_mutex; + const bool m_isatty; + std::ostream& m_outputStream; +#ifdef _WIN32 + HANDLE m_outputHandle; +#endif + }; +} diff --git a/include/plog/Appenders/DebugOutputAppender.h b/include/plog/Appenders/DebugOutputAppender.h new file mode 100644 index 000000000..5d7c95ef2 --- /dev/null +++ b/include/plog/Appenders/DebugOutputAppender.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +namespace plog +{ + template + class PLOG_LINKAGE_HIDDEN DebugOutputAppender : public IAppender + { + public: + virtual void write(const Record& record) PLOG_OVERRIDE + { + OutputDebugStringW(util::toWide(Formatter::format(record)).c_str()); + } + }; +} diff --git a/include/plog/Appenders/DynamicAppender.h b/include/plog/Appenders/DynamicAppender.h new file mode 100644 index 000000000..f078c08cc --- /dev/null +++ b/include/plog/Appenders/DynamicAppender.h @@ -0,0 +1,42 @@ +#pragma once +#include +#include + +namespace plog +{ + class PLOG_LINKAGE_HIDDEN DynamicAppender : public IAppender + { + public: + DynamicAppender& addAppender(IAppender* appender) + { + assert(appender != this); + + util::MutexLock lock(m_mutex); + m_appenders.insert(appender); + + return *this; + } + + DynamicAppender& removeAppender(IAppender* appender) + { + util::MutexLock lock(m_mutex); + m_appenders.erase(appender); + + return *this; + } + + virtual void write(const Record& record) PLOG_OVERRIDE + { + util::MutexLock lock(m_mutex); + + for (std::set::iterator it = m_appenders.begin(); it != m_appenders.end(); ++it) + { + (*it)->write(record); + } + } + + private: + mutable util::Mutex m_mutex; + std::set m_appenders; + }; +} diff --git a/include/plog/Appenders/EventLogAppender.h b/include/plog/Appenders/EventLogAppender.h new file mode 100644 index 000000000..3b7065be1 --- /dev/null +++ b/include/plog/Appenders/EventLogAppender.h @@ -0,0 +1,117 @@ +#pragma once +#include +#include + +namespace plog +{ + template + class PLOG_LINKAGE_HIDDEN EventLogAppender : public IAppender + { + public: + EventLogAppender(const util::nchar* sourceName) : m_eventSource(RegisterEventSourceW(NULL, util::toWide(sourceName).c_str())) + { + } + + ~EventLogAppender() + { + DeregisterEventSource(m_eventSource); + } + + virtual void write(const Record& record) PLOG_OVERRIDE + { + util::nstring str = Formatter::format(record); + + write(record.getSeverity(), util::toWide(str).c_str()); + } + + private: + void write(Severity severity, const wchar_t* str) + { + const wchar_t* logMessagePtr[] = { str }; + + ReportEventW(m_eventSource, logSeverityToType(severity), static_cast(severity), 0, NULL, 1, 0, logMessagePtr, NULL); + } + + static WORD logSeverityToType(plog::Severity severity) + { + switch (severity) + { + case plog::fatal: + case plog::error: + return eventLog::kErrorType; + + case plog::warning: + return eventLog::kWarningType; + + case plog::info: + case plog::debug: + case plog::verbose: + default: + return eventLog::kInformationType; + } + } + + private: + HANDLE m_eventSource; + }; + + class EventLogAppenderRegistry + { + public: + static bool add(const util::nchar* sourceName, const util::nchar* logName = PLOG_NSTR("Application")) + { + std::wstring logKeyName; + std::wstring sourceKeyName; + getKeyNames(sourceName, logName, sourceKeyName, logKeyName); + + HKEY sourceKey; + if (0 != RegCreateKeyExW(hkey::kLocalMachine, sourceKeyName.c_str(), 0, NULL, 0, regSam::kSetValue, NULL, &sourceKey, NULL)) + { + return false; + } + + const DWORD kTypesSupported = eventLog::kErrorType | eventLog::kWarningType | eventLog::kInformationType; + RegSetValueExW(sourceKey, L"TypesSupported", 0, regType::kDword, reinterpret_cast(&kTypesSupported), sizeof(kTypesSupported)); + + const wchar_t kEventMessageFile[] = L"%windir%\\Microsoft.NET\\Framework\\v4.0.30319\\EventLogMessages.dll;%windir%\\Microsoft.NET\\Framework\\v2.0.50727\\EventLogMessages.dll"; + RegSetValueExW(sourceKey, L"EventMessageFile", 0, regType::kExpandSz, reinterpret_cast(kEventMessageFile), sizeof(kEventMessageFile) - sizeof(*kEventMessageFile)); + + RegCloseKey(sourceKey); + return true; + } + + static bool exists(const util::nchar* sourceName, const util::nchar* logName = PLOG_NSTR("Application")) + { + std::wstring logKeyName; + std::wstring sourceKeyName; + getKeyNames(sourceName, logName, sourceKeyName, logKeyName); + + HKEY sourceKey; + if (0 != RegOpenKeyExW(hkey::kLocalMachine, sourceKeyName.c_str(), 0, regSam::kQueryValue, &sourceKey)) + { + return false; + } + + RegCloseKey(sourceKey); + return true; + } + + static void remove(const util::nchar* sourceName, const util::nchar* logName = PLOG_NSTR("Application")) + { + std::wstring logKeyName; + std::wstring sourceKeyName; + getKeyNames(sourceName, logName, sourceKeyName, logKeyName); + + RegDeleteKeyW(hkey::kLocalMachine, sourceKeyName.c_str()); + RegDeleteKeyW(hkey::kLocalMachine, logKeyName.c_str()); + } + + private: + static void getKeyNames(const util::nchar* sourceName, const util::nchar* logName, std::wstring& sourceKeyName, std::wstring& logKeyName) + { + const std::wstring kPrefix = L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\"; + logKeyName = kPrefix + util::toWide(logName); + sourceKeyName = logKeyName + L"\\" + util::toWide(sourceName); + } + }; +} diff --git a/include/plog/Appenders/IAppender.h b/include/plog/Appenders/IAppender.h new file mode 100644 index 000000000..56b7827ae --- /dev/null +++ b/include/plog/Appenders/IAppender.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +namespace plog +{ + class PLOG_LINKAGE IAppender + { + public: + virtual ~IAppender() + { + } + + virtual void write(const Record& record) = 0; + }; +} diff --git a/include/plog/Appenders/RollingFileAppender.h b/include/plog/Appenders/RollingFileAppender.h new file mode 100644 index 000000000..3b667287a --- /dev/null +++ b/include/plog/Appenders/RollingFileAppender.h @@ -0,0 +1,148 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace plog +{ + template > + class PLOG_LINKAGE_HIDDEN RollingFileAppender : public IAppender + { + public: + RollingFileAppender(const util::nchar* fileName, size_t maxFileSize = 0, int maxFiles = 0) + : m_fileSize() + , m_maxFileSize() + , m_maxFiles(maxFiles) + , m_firstWrite(true) + { + setFileName(fileName); + setMaxFileSize(maxFileSize); + } + +#if defined(_WIN32) && !PLOG_CHAR_IS_UTF8 + RollingFileAppender(const char* fileName, size_t maxFileSize = 0, int maxFiles = 0) + : m_fileSize() + , m_maxFileSize() + , m_maxFiles(maxFiles) + , m_firstWrite(true) + { + setFileName(fileName); + setMaxFileSize(maxFileSize); + } +#endif + + virtual void write(const Record& record) PLOG_OVERRIDE + { + util::MutexLock lock(m_mutex); + + if (m_firstWrite) + { + openLogFile(); + m_firstWrite = false; + } + else if (m_maxFiles > 0 && m_fileSize > m_maxFileSize && static_cast(-1) != m_fileSize) + { + rollLogFiles(); + } + + size_t bytesWritten = m_file.write(Converter::convert(Formatter::format(record))); + + if (static_cast(-1) != bytesWritten) + { + m_fileSize += bytesWritten; + } + } + + void setFileName(const util::nchar* fileName) + { + util::MutexLock lock(m_mutex); + + util::splitFileName(fileName, m_fileNameNoExt, m_fileExt); + + m_file.close(); + m_firstWrite = true; + } + +#if defined(_WIN32) && !PLOG_CHAR_IS_UTF8 + void setFileName(const char* fileName) + { + setFileName(util::toWide(fileName).c_str()); + } +#endif + + void setMaxFiles(int maxFiles) + { + m_maxFiles = maxFiles; + } + + void setMaxFileSize(size_t maxFileSize) + { + m_maxFileSize = (std::max)(maxFileSize, static_cast(1000)); // set a lower limit for the maxFileSize + } + + void rollLogFiles() + { + m_file.close(); + + util::nstring lastFileName = buildFileName(m_maxFiles - 1); + util::File::unlink(lastFileName); + + for (int fileNumber = m_maxFiles - 2; fileNumber >= 0; --fileNumber) + { + util::nstring currentFileName = buildFileName(fileNumber); + util::nstring nextFileName = buildFileName(fileNumber + 1); + + util::File::rename(currentFileName, nextFileName); + } + + openLogFile(); + m_firstWrite = false; + } + + private: + void openLogFile() + { + m_fileSize = m_file.open(buildFileName()); + + if (0 == m_fileSize) + { + size_t bytesWritten = m_file.write(Converter::header(Formatter::header())); + + if (static_cast(-1) != bytesWritten) + { + m_fileSize += bytesWritten; + } + } + } + + util::nstring buildFileName(int fileNumber = 0) + { + util::nostringstream ss; + ss << m_fileNameNoExt; + + if (fileNumber > 0) + { + ss << '.' << fileNumber; + } + + if (!m_fileExt.empty()) + { + ss << '.' << m_fileExt; + } + + return ss.str(); + } + + private: + util::Mutex m_mutex; + util::File m_file; + size_t m_fileSize; + size_t m_maxFileSize; + int m_maxFiles; + util::nstring m_fileExt; + util::nstring m_fileNameNoExt; + bool m_firstWrite; + }; +} diff --git a/include/plog/Converters/NativeEOLConverter.h b/include/plog/Converters/NativeEOLConverter.h new file mode 100644 index 000000000..a395e4e89 --- /dev/null +++ b/include/plog/Converters/NativeEOLConverter.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include + +namespace plog +{ + template + class NativeEOLConverter : public NextConverter + { +#ifdef _WIN32 + public: + static std::string header(const util::nstring& str) + { + return NextConverter::header(fixLineEndings(str)); + } + + static std::string convert(const util::nstring& str) + { + return NextConverter::convert(fixLineEndings(str)); + } + + private: + static util::nstring fixLineEndings(const util::nstring& str) + { + util::nstring output; + output.reserve(str.length() * 2); // the worst case requires 2x chars + + for (size_t i = 0; i < str.size(); ++i) + { + util::nchar ch = str[i]; + + if (ch == PLOG_NSTR('\n')) + { + output.push_back(PLOG_NSTR('\r')); + } + + output.push_back(ch); + } + + return output; + } +#endif + }; +} diff --git a/include/plog/Converters/UTF8Converter.h b/include/plog/Converters/UTF8Converter.h new file mode 100644 index 000000000..9de5a6303 --- /dev/null +++ b/include/plog/Converters/UTF8Converter.h @@ -0,0 +1,28 @@ +#pragma once +#include + +namespace plog +{ + class UTF8Converter + { + public: + static std::string header(const util::nstring& str) + { + const char kBOM[] = "\xEF\xBB\xBF"; + + return std::string(kBOM) + convert(str); + } + +#if PLOG_CHAR_IS_UTF8 + static const std::string& convert(const util::nstring& str) + { + return str; + } +#else + static std::string convert(const util::nstring& str) + { + return util::toNarrow(str, codePage::kUTF8); + } +#endif + }; +} diff --git a/include/plog/Formatters/CsvFormatter.h b/include/plog/Formatters/CsvFormatter.h new file mode 100644 index 000000000..282c57f19 --- /dev/null +++ b/include/plog/Formatters/CsvFormatter.h @@ -0,0 +1,57 @@ +#pragma once +#include +#include +#include + +namespace plog +{ + template + class CsvFormatterImpl + { + public: + static util::nstring header() + { + return PLOG_NSTR("Date;Time;Severity;TID;This;Function;Message\n"); + } + + static util::nstring format(const Record& record) + { + tm t; + useUtcTime ? util::gmtime_s(&t, &record.getTime().time) : util::localtime_s(&t, &record.getTime().time); + + util::nostringstream ss; + ss << t.tm_year + 1900 << PLOG_NSTR("/") << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_mon + 1 << PLOG_NSTR("/") << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_mday << PLOG_NSTR(";"); + ss << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_hour << PLOG_NSTR(":") << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_min << PLOG_NSTR(":") << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_sec << PLOG_NSTR(".") << std::setfill(PLOG_NSTR('0')) << std::setw(3) << static_cast (record.getTime().millitm) << PLOG_NSTR(";"); + ss << severityToString(record.getSeverity()) << PLOG_NSTR(";"); + ss << record.getTid() << PLOG_NSTR(";"); + ss << record.getObject() << PLOG_NSTR(";"); + ss << record.getFunc() << PLOG_NSTR("@") << record.getLine() << PLOG_NSTR(";"); + + util::nstring message = record.getMessage(); + + if (message.size() > kMaxMessageSize) + { + message.resize(kMaxMessageSize); + message.append(PLOG_NSTR("...")); + } + + util::nistringstream split(message); + util::nstring token; + + while (!split.eof()) + { + std::getline(split, token, PLOG_NSTR('"')); + ss << PLOG_NSTR("\"") << token << PLOG_NSTR("\""); + } + + ss << PLOG_NSTR("\n"); + + return ss.str(); + } + + static const size_t kMaxMessageSize = 32000; + }; + + class CsvFormatter : public CsvFormatterImpl {}; + class CsvFormatterUtcTime : public CsvFormatterImpl {}; +} diff --git a/include/plog/Formatters/FuncMessageFormatter.h b/include/plog/Formatters/FuncMessageFormatter.h new file mode 100644 index 000000000..aa57806b8 --- /dev/null +++ b/include/plog/Formatters/FuncMessageFormatter.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include + +namespace plog +{ + class FuncMessageFormatter + { + public: + static util::nstring header() + { + return util::nstring(); + } + + static util::nstring format(const Record& record) + { + util::nostringstream ss; + ss << record.getFunc() << PLOG_NSTR("@") << record.getLine() << PLOG_NSTR(": ") << record.getMessage() << PLOG_NSTR("\n"); + + return ss.str(); + } + }; +} diff --git a/include/plog/Formatters/MessageOnlyFormatter.h b/include/plog/Formatters/MessageOnlyFormatter.h new file mode 100644 index 000000000..b2db5a5a0 --- /dev/null +++ b/include/plog/Formatters/MessageOnlyFormatter.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include + +namespace plog +{ + class MessageOnlyFormatter + { + public: + static util::nstring header() + { + return util::nstring(); + } + + static util::nstring format(const Record& record) + { + util::nostringstream ss; + ss << record.getMessage() << PLOG_NSTR("\n"); + + return ss.str(); + } + }; +} diff --git a/include/plog/Formatters/TxtFormatter.h b/include/plog/Formatters/TxtFormatter.h new file mode 100644 index 000000000..48e2d50b8 --- /dev/null +++ b/include/plog/Formatters/TxtFormatter.h @@ -0,0 +1,36 @@ +#pragma once +#include +#include +#include + +namespace plog +{ + template + class TxtFormatterImpl + { + public: + static util::nstring header() + { + return util::nstring(); + } + + static util::nstring format(const Record& record) + { + tm t; + useUtcTime ? util::gmtime_s(&t, &record.getTime().time) : util::localtime_s(&t, &record.getTime().time); + + util::nostringstream ss; + ss << t.tm_year + 1900 << "-" << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_mon + 1 << PLOG_NSTR("-") << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_mday << PLOG_NSTR(" "); + ss << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_hour << PLOG_NSTR(":") << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_min << PLOG_NSTR(":") << std::setfill(PLOG_NSTR('0')) << std::setw(2) << t.tm_sec << PLOG_NSTR(".") << std::setfill(PLOG_NSTR('0')) << std::setw(3) << static_cast (record.getTime().millitm) << PLOG_NSTR(" "); + ss << std::setfill(PLOG_NSTR(' ')) << std::setw(5) << std::left << severityToString(record.getSeverity()) << PLOG_NSTR(" "); + ss << PLOG_NSTR("[") << record.getTid() << PLOG_NSTR("] "); + ss << PLOG_NSTR("[") << record.getFunc() << PLOG_NSTR("@") << record.getLine() << PLOG_NSTR("] "); + ss << record.getMessage() << PLOG_NSTR("\n"); + + return ss.str(); + } + }; + + class TxtFormatter : public TxtFormatterImpl {}; + class TxtFormatterUtcTime : public TxtFormatterImpl {}; +} diff --git a/include/plog/Helpers/AscDump.h b/include/plog/Helpers/AscDump.h new file mode 100644 index 000000000..18b9a6f4f --- /dev/null +++ b/include/plog/Helpers/AscDump.h @@ -0,0 +1,40 @@ +#pragma once +#include +#include + +namespace plog +{ + class AscDump + { + public: + AscDump(const void* ptr, size_t size) + : m_ptr(static_cast(ptr)) + , m_size(size) + { + } + + friend util::nostringstream& operator<<(util::nostringstream& stream, const AscDump& ascDump); + + private: + const char* m_ptr; + size_t m_size; + }; + + inline util::nostringstream& operator<<(util::nostringstream& stream, const AscDump& ascDump) + { + for (size_t i = 0; i < ascDump.m_size; ++i) + { + stream << (std::isprint(ascDump.m_ptr[i]) ? ascDump.m_ptr[i] : '.'); + } + + return stream; + } + + inline AscDump ascdump(const void* ptr, size_t size) { return AscDump(ptr, size); } + + template + inline AscDump ascdump(const Container& container) { return AscDump(container.data(), container.size() * sizeof(*container.data())); } + + template + inline AscDump ascdump(const T (&arr)[N]) { return AscDump(arr, N * sizeof(*arr)); } +} diff --git a/include/plog/Helpers/HexDump.h b/include/plog/Helpers/HexDump.h new file mode 100644 index 000000000..b0493d707 --- /dev/null +++ b/include/plog/Helpers/HexDump.h @@ -0,0 +1,79 @@ +#pragma once +#include +#include + +namespace plog +{ + class HexDump + { + public: + HexDump(const void* ptr, size_t size) + : m_ptr(static_cast(ptr)) + , m_size(size) + , m_group(8) + , m_digitSeparator(" ") + , m_groupSeparator(" ") + { + } + + HexDump& group(size_t group) + { + m_group = group; + return *this; + } + + HexDump& separator(const char* digitSeparator) + { + m_digitSeparator = digitSeparator; + return *this; + } + + HexDump& separator(const char* digitSeparator, const char* groupSeparator) + { + m_digitSeparator = digitSeparator; + m_groupSeparator = groupSeparator; + return *this; + } + + friend util::nostringstream& operator<<(util::nostringstream& stream, const HexDump&); + + private: + const unsigned char* m_ptr; + size_t m_size; + size_t m_group; + const char* m_digitSeparator; + const char* m_groupSeparator; + }; + + inline util::nostringstream& operator<<(util::nostringstream& stream, const HexDump& hexDump) + { + stream << std::hex << std::setfill(PLOG_NSTR('0')); + + for (size_t i = 0; i < hexDump.m_size;) + { + stream << std::setw(2) << static_cast(hexDump.m_ptr[i]); + + if (++i < hexDump.m_size) + { + if (hexDump.m_group > 0 && i % hexDump.m_group == 0) + { + stream << hexDump.m_groupSeparator; + } + else + { + stream << hexDump.m_digitSeparator; + } + } + } + + return stream; + } + + inline HexDump hexdump(const void* ptr, size_t size) { return HexDump(ptr, size); } + + template + inline HexDump hexdump(const Container& container) { return HexDump(container.data(), container.size() * sizeof(*container.data())); } + + template + inline HexDump hexdump(const T (&arr)[N]) { return HexDump(arr, N * sizeof(*arr)); } +} diff --git a/include/plog/Helpers/PrintVar.h b/include/plog/Helpers/PrintVar.h new file mode 100644 index 000000000..465e1938f --- /dev/null +++ b/include/plog/Helpers/PrintVar.h @@ -0,0 +1,24 @@ +#pragma once + +#define PLOG_IMPL_PRINT_VAR_1(a1) #a1 ": " << a1 +#define PLOG_IMPL_PRINT_VAR_2(a1, a2) PLOG_IMPL_PRINT_VAR_1(a1) PLOG_IMPL_PRINT_VAR_TAIL(a2) +#define PLOG_IMPL_PRINT_VAR_3(a1, a2, a3) PLOG_IMPL_PRINT_VAR_2(a1, a2) PLOG_IMPL_PRINT_VAR_TAIL(a3) +#define PLOG_IMPL_PRINT_VAR_4(a1, a2, a3, a4) PLOG_IMPL_PRINT_VAR_3(a1, a2, a3) PLOG_IMPL_PRINT_VAR_TAIL(a4) +#define PLOG_IMPL_PRINT_VAR_5(a1, a2, a3, a4, a5) PLOG_IMPL_PRINT_VAR_4(a1, a2, a3, a4) PLOG_IMPL_PRINT_VAR_TAIL(a5) +#define PLOG_IMPL_PRINT_VAR_6(a1, a2, a3, a4, a5, a6) PLOG_IMPL_PRINT_VAR_5(a1, a2, a3, a4, a5) PLOG_IMPL_PRINT_VAR_TAIL(a6) +#define PLOG_IMPL_PRINT_VAR_7(a1, a2, a3, a4, a5, a6, a7) PLOG_IMPL_PRINT_VAR_6(a1, a2, a3, a4, a5, a6) PLOG_IMPL_PRINT_VAR_TAIL(a7) +#define PLOG_IMPL_PRINT_VAR_8(a1, a2, a3, a4, a5, a6, a7, a8) PLOG_IMPL_PRINT_VAR_7(a1, a2, a3, a4, a5, a6, a7) PLOG_IMPL_PRINT_VAR_TAIL(a8) +#define PLOG_IMPL_PRINT_VAR_9(a1, a2, a3, a4, a5, a6, a7, a8, a9) PLOG_IMPL_PRINT_VAR_8(a1, a2, a3, a4, a5, a6, a7, a8) PLOG_IMPL_PRINT_VAR_TAIL(a9) +#define PLOG_IMPL_PRINT_VAR_TAIL(a) << ", " PLOG_IMPL_PRINT_VAR_1(a) + +#define PLOG_IMPL_PRINT_VAR_EXPAND(x) x + +#ifdef __GNUC__ +#pragma GCC system_header // disable warning: variadic macros are a C99 feature +#endif + +#define PLOG_IMPL_PRINT_VAR_GET_MACRO(x1, x2, x3, x4, x5, x6, x7, x8, x9, NAME, ...) NAME + +#define PLOG_PRINT_VAR(...) PLOG_IMPL_PRINT_VAR_EXPAND(PLOG_IMPL_PRINT_VAR_GET_MACRO(__VA_ARGS__,\ + PLOG_IMPL_PRINT_VAR_9, PLOG_IMPL_PRINT_VAR_8, PLOG_IMPL_PRINT_VAR_7, PLOG_IMPL_PRINT_VAR_6, PLOG_IMPL_PRINT_VAR_5,\ + PLOG_IMPL_PRINT_VAR_4, PLOG_IMPL_PRINT_VAR_3, PLOG_IMPL_PRINT_VAR_2, PLOG_IMPL_PRINT_VAR_1, UNUSED)(__VA_ARGS__)) diff --git a/include/plog/Init.h b/include/plog/Init.h new file mode 100644 index 000000000..615c41d6c --- /dev/null +++ b/include/plog/Init.h @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace plog +{ + template + PLOG_LINKAGE_HIDDEN inline Logger& init(Severity maxSeverity = none, IAppender* appender = NULL) + { + static Logger logger(maxSeverity); + return appender ? logger.addAppender(appender) : logger; + } + + inline Logger& init(Severity maxSeverity = none, IAppender* appender = NULL) + { + return init(maxSeverity, appender); + } +} diff --git a/include/plog/Initializers/ConsoleInitializer.h b/include/plog/Initializers/ConsoleInitializer.h new file mode 100644 index 000000000..5b504abd5 --- /dev/null +++ b/include/plog/Initializers/ConsoleInitializer.h @@ -0,0 +1,22 @@ +#pragma once +#include +#include + +namespace plog +{ + ////////////////////////////////////////////////////////////////////////// + // ColorConsoleAppender with any Formatter + + template + PLOG_LINKAGE_HIDDEN inline Logger& init(Severity maxSeverity, OutputStream outputStream) + { + static ColorConsoleAppender appender(outputStream); + return init(maxSeverity, &appender); + } + + template + inline Logger& init(Severity maxSeverity, OutputStream outputStream) + { + return init(maxSeverity, outputStream); + } +} diff --git a/include/plog/Initializers/RollingFileInitializer.h b/include/plog/Initializers/RollingFileInitializer.h new file mode 100644 index 000000000..dc0adda14 --- /dev/null +++ b/include/plog/Initializers/RollingFileInitializer.h @@ -0,0 +1,80 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace plog +{ + ////////////////////////////////////////////////////////////////////////// + // RollingFileAppender with any Formatter + + template + PLOG_LINKAGE_HIDDEN inline Logger& init(Severity maxSeverity, const util::nchar* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + static RollingFileAppender rollingFileAppender(fileName, maxFileSize, maxFiles); + return init(maxSeverity, &rollingFileAppender); + } + + template + inline Logger& init(Severity maxSeverity, const util::nchar* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + return init(maxSeverity, fileName, maxFileSize, maxFiles); + } + + ////////////////////////////////////////////////////////////////////////// + // RollingFileAppender with TXT/CSV chosen by file extension + + namespace + { + inline bool isCsv(const util::nchar* fileName) + { + const util::nchar* dot = util::findExtensionDot(fileName); +#if PLOG_CHAR_IS_UTF8 + return dot && 0 == std::strcmp(dot, ".csv"); +#else + return dot && 0 == std::wcscmp(dot, L".csv"); +#endif + } + } + + template + inline Logger& init(Severity maxSeverity, const util::nchar* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + return isCsv(fileName) ? init(maxSeverity, fileName, maxFileSize, maxFiles) : init(maxSeverity, fileName, maxFileSize, maxFiles); + } + + inline Logger& init(Severity maxSeverity, const util::nchar* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + return init(maxSeverity, fileName, maxFileSize, maxFiles); + } + + ////////////////////////////////////////////////////////////////////////// + // CHAR variants for Windows + +#if defined(_WIN32) && !PLOG_CHAR_IS_UTF8 + template + inline Logger& init(Severity maxSeverity, const char* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + return init(maxSeverity, util::toWide(fileName).c_str(), maxFileSize, maxFiles); + } + + template + inline Logger& init(Severity maxSeverity, const char* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + return init(maxSeverity, fileName, maxFileSize, maxFiles); + } + + template + inline Logger& init(Severity maxSeverity, const char* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + return init(maxSeverity, util::toWide(fileName).c_str(), maxFileSize, maxFiles); + } + + inline Logger& init(Severity maxSeverity, const char* fileName, size_t maxFileSize = 0, int maxFiles = 0) + { + return init(maxSeverity, fileName, maxFileSize, maxFiles); + } +#endif +} diff --git a/include/plog/LICENSE b/include/plog/LICENSE new file mode 100644 index 000000000..8a91a0aee --- /dev/null +++ b/include/plog/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Sergey Podobry + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/include/plog/Log.h b/include/plog/Log.h new file mode 100644 index 000000000..127cb9d6b --- /dev/null +++ b/include/plog/Log.h @@ -0,0 +1,211 @@ +////////////////////////////////////////////////////////////////////////// +// Plog - portable and simple log for C++ +// Documentation and sources: https://github.com/SergiusTheBest/plog +// License: MIT, https://choosealicense.com/licenses/mit + +#pragma once +#include + +////////////////////////////////////////////////////////////////////////// +// Helper macros that get context info + +#if defined(PLOG_ENABLE_GET_THIS) && defined(_MSC_VER) && _MSC_VER >= 1600 && !defined(__INTELLISENSE__) && !defined(__INTEL_COMPILER) && !defined(__llvm__) && !defined(__RESHARPER__) // >= Visual Studio 2010, skip IntelliSense, Intel Compiler, Clang Code Model and ReSharper +# define PLOG_GET_THIS() __if_exists(this) { this } __if_not_exists(this) { 0 } +#else +# define PLOG_GET_THIS() reinterpret_cast(0) +#endif + +#ifdef PLOG_NO_CAPTURE_FUNCTION_NAME +# define PLOG_GET_FUNC() "" +#elif defined(_MSC_VER) +# define PLOG_GET_FUNC() __FUNCTION__ +#elif defined(__BORLANDC__) +# define PLOG_GET_FUNC() __FUNC__ +#else +# define PLOG_GET_FUNC() __PRETTY_FUNCTION__ +#endif + +#ifdef PLOG_CAPTURE_FILE +# define PLOG_GET_FILE() __FILE__ +#else +# define PLOG_GET_FILE() "" +#endif + +////////////////////////////////////////////////////////////////////////// +// Log severity level checker + +#ifdef PLOG_DISABLE_LOGGING +# ifdef _MSC_VER +# define IF_PLOG_(instanceId, severity) __pragma(warning(push)) __pragma(warning(disable:4127)) if (true) {;} else __pragma(warning(pop)) // conditional expression is constant +# else +# define IF_PLOG_(instanceId, severity) if (true) {;} else +# endif +#else +# define IF_PLOG_(instanceId, severity) if (!plog::get() || !plog::get()->checkSeverity(severity)) {;} else +#endif + +#define IF_PLOG(severity) IF_PLOG_(PLOG_DEFAULT_INSTANCE_ID, severity) + +////////////////////////////////////////////////////////////////////////// +// Main logging macros + +#ifdef PLOG_MESSAGE_PREFIX +# define PLOG_PRINT_MESSAGE_PREFIX() << PLOG_MESSAGE_PREFIX +#else +# define PLOG_PRINT_MESSAGE_PREFIX() +#endif + +#define PLOG_(instanceId, severity) IF_PLOG_(instanceId, severity) (*plog::get()) += \ + plog::Record(severity, PLOG_GET_FUNC(), __LINE__, PLOG_GET_FILE(), PLOG_GET_THIS(), instanceId).ref() PLOG_PRINT_MESSAGE_PREFIX() +#define PLOG(severity) PLOG_(PLOG_DEFAULT_INSTANCE_ID, severity) + +#define PLOG_VERBOSE PLOG(plog::verbose) +#define PLOG_DEBUG PLOG(plog::debug) +#define PLOG_INFO PLOG(plog::info) +#define PLOG_WARNING PLOG(plog::warning) +#define PLOG_ERROR PLOG(plog::error) +#define PLOG_FATAL PLOG(plog::fatal) +#define PLOG_NONE PLOG(plog::none) + +#define PLOG_VERBOSE_(instanceId) PLOG_(instanceId, plog::verbose) +#define PLOG_DEBUG_(instanceId) PLOG_(instanceId, plog::debug) +#define PLOG_INFO_(instanceId) PLOG_(instanceId, plog::info) +#define PLOG_WARNING_(instanceId) PLOG_(instanceId, plog::warning) +#define PLOG_ERROR_(instanceId) PLOG_(instanceId, plog::error) +#define PLOG_FATAL_(instanceId) PLOG_(instanceId, plog::fatal) +#define PLOG_NONE_(instanceId) PLOG_(instanceId, plog::none) + +#define PLOGV PLOG_VERBOSE +#define PLOGD PLOG_DEBUG +#define PLOGI PLOG_INFO +#define PLOGW PLOG_WARNING +#define PLOGE PLOG_ERROR +#define PLOGF PLOG_FATAL +#define PLOGN PLOG_NONE + +#define PLOGV_(instanceId) PLOG_VERBOSE_(instanceId) +#define PLOGD_(instanceId) PLOG_DEBUG_(instanceId) +#define PLOGI_(instanceId) PLOG_INFO_(instanceId) +#define PLOGW_(instanceId) PLOG_WARNING_(instanceId) +#define PLOGE_(instanceId) PLOG_ERROR_(instanceId) +#define PLOGF_(instanceId) PLOG_FATAL_(instanceId) +#define PLOGN_(instanceId) PLOG_NONE_(instanceId) + +////////////////////////////////////////////////////////////////////////// +// Conditional logging macros + +#define PLOG_IF_(instanceId, severity, condition) if (!(condition)) {;} else PLOG_(instanceId, severity) +#define PLOG_IF(severity, condition) PLOG_IF_(PLOG_DEFAULT_INSTANCE_ID, severity, condition) + +#define PLOG_VERBOSE_IF(condition) PLOG_IF(plog::verbose, condition) +#define PLOG_DEBUG_IF(condition) PLOG_IF(plog::debug, condition) +#define PLOG_INFO_IF(condition) PLOG_IF(plog::info, condition) +#define PLOG_WARNING_IF(condition) PLOG_IF(plog::warning, condition) +#define PLOG_ERROR_IF(condition) PLOG_IF(plog::error, condition) +#define PLOG_FATAL_IF(condition) PLOG_IF(plog::fatal, condition) +#define PLOG_NONE_IF(condition) PLOG_IF(plog::none, condition) + +#define PLOG_VERBOSE_IF_(instanceId, condition) PLOG_IF_(instanceId, plog::verbose, condition) +#define PLOG_DEBUG_IF_(instanceId, condition) PLOG_IF_(instanceId, plog::debug, condition) +#define PLOG_INFO_IF_(instanceId, condition) PLOG_IF_(instanceId, plog::info, condition) +#define PLOG_WARNING_IF_(instanceId, condition) PLOG_IF_(instanceId, plog::warning, condition) +#define PLOG_ERROR_IF_(instanceId, condition) PLOG_IF_(instanceId, plog::error, condition) +#define PLOG_FATAL_IF_(instanceId, condition) PLOG_IF_(instanceId, plog::fatal, condition) +#define PLOG_NONE_IF_(instanceId, condition) PLOG_IF_(instanceId, plog::none, condition) + +#define PLOGV_IF(condition) PLOG_VERBOSE_IF(condition) +#define PLOGD_IF(condition) PLOG_DEBUG_IF(condition) +#define PLOGI_IF(condition) PLOG_INFO_IF(condition) +#define PLOGW_IF(condition) PLOG_WARNING_IF(condition) +#define PLOGE_IF(condition) PLOG_ERROR_IF(condition) +#define PLOGF_IF(condition) PLOG_FATAL_IF(condition) +#define PLOGN_IF(condition) PLOG_NONE_IF(condition) + +#define PLOGV_IF_(instanceId, condition) PLOG_VERBOSE_IF_(instanceId, condition) +#define PLOGD_IF_(instanceId, condition) PLOG_DEBUG_IF_(instanceId, condition) +#define PLOGI_IF_(instanceId, condition) PLOG_INFO_IF_(instanceId, condition) +#define PLOGW_IF_(instanceId, condition) PLOG_WARNING_IF_(instanceId, condition) +#define PLOGE_IF_(instanceId, condition) PLOG_ERROR_IF_(instanceId, condition) +#define PLOGF_IF_(instanceId, condition) PLOG_FATAL_IF_(instanceId, condition) +#define PLOGN_IF_(instanceId, condition) PLOG_NONE_IF_(instanceId, condition) + +// Old macro names for downward compatibility. To bypass including these macro names, add +// #define PLOG_OMIT_LOG_DEFINES before #include +#ifndef PLOG_OMIT_LOG_DEFINES + +////////////////////////////////////////////////////////////////////////// +// Main logging macros - can be changed later to point at macros for a different logging package + +#define LOG_(instanceId, severity) PLOG_(instanceId, severity) +#define LOG(severity) PLOG(severity) + +#define LOG_VERBOSE PLOG_VERBOSE +#define LOG_DEBUG PLOG_DEBUG +#define LOG_INFO PLOG_INFO +#define LOG_WARNING PLOG_WARNING +#define LOG_ERROR PLOG_ERROR +#define LOG_FATAL PLOG_FATAL +#define LOG_NONE PLOG_NONE + +#define LOG_VERBOSE_(instanceId) PLOG_VERBOSE_(instanceId) +#define LOG_DEBUG_(instanceId) PLOG_DEBUG_(instanceId) +#define LOG_INFO_(instanceId) PLOG_INFO_(instanceId) +#define LOG_WARNING_(instanceId) PLOG_WARNING_(instanceId) +#define LOG_ERROR_(instanceId) PLOG_ERROR_(instanceId) +#define LOG_FATAL_(instanceId) PLOG_FATAL_(instanceId) +#define LOG_NONE_(instanceId) PLOG_NONE_(instanceId) + +#define LOGV PLOGV +#define LOGD PLOGD +#define LOGI PLOGI +#define LOGW PLOGW +#define LOGE PLOGE +#define LOGF PLOGF +#define LOGN PLOGN + +#define LOGV_(instanceId) PLOGV_(instanceId) +#define LOGD_(instanceId) PLOGD_(instanceId) +#define LOGI_(instanceId) PLOGI_(instanceId) +#define LOGW_(instanceId) PLOGW_(instanceId) +#define LOGE_(instanceId) PLOGE_(instanceId) +#define LOGF_(instanceId) PLOGF_(instanceId) +#define LOGN_(instanceId) PLOGN_(instanceId) + +////////////////////////////////////////////////////////////////////////// +// Conditional logging macros + +#define LOG_IF_(instanceId, severity, condition) PLOG_IF_(instanceId, severity, condition) +#define LOG_IF(severity, condition) PLOG_IF(severity, condition) + +#define LOG_VERBOSE_IF(condition) PLOG_VERBOSE_IF(condition) +#define LOG_DEBUG_IF(condition) PLOG_DEBUG_IF(condition) +#define LOG_INFO_IF(condition) PLOG_INFO_IF(condition) +#define LOG_WARNING_IF(condition) PLOG_WARNING_IF(condition) +#define LOG_ERROR_IF(condition) PLOG_ERROR_IF(condition) +#define LOG_FATAL_IF(condition) PLOG_FATAL_IF(condition) +#define LOG_NONE_IF(condition) PLOG_NONE_IF(condition) + +#define LOG_VERBOSE_IF_(instanceId, condition) PLOG_VERBOSE_IF_(instanceId, condition) +#define LOG_DEBUG_IF_(instanceId, condition) PLOG_DEBUG_IF_(instanceId, condition) +#define LOG_INFO_IF_(instanceId, condition) PLOG_INFO_IF_(instanceId, condition) +#define LOG_WARNING_IF_(instanceId, condition) PLOG_WARNING_IF_(instanceId, condition) +#define LOG_ERROR_IF_(instanceId, condition) PLOG_ERROR_IF_(instanceId, condition) +#define LOG_FATAL_IF_(instanceId, condition) PLOG_FATAL_IF_(instanceId, condition) +#define LOG_NONE_IF_(instanceId, condition) PLOG_NONE_IF_(instanceId, condition) + +#define LOGV_IF(condition) PLOGV_IF(condition) +#define LOGD_IF(condition) PLOGD_IF(condition) +#define LOGI_IF(condition) PLOGI_IF(condition) +#define LOGW_IF(condition) PLOGW_IF(condition) +#define LOGE_IF(condition) PLOGE_IF(condition) +#define LOGF_IF(condition) PLOGF_IF(condition) +#define LOGN_IF(condition) PLOGN_IF(condition) + +#define LOGV_IF_(instanceId, condition) PLOGV_IF_(instanceId, condition) +#define LOGD_IF_(instanceId, condition) PLOGD_IF_(instanceId, condition) +#define LOGI_IF_(instanceId, condition) PLOGI_IF_(instanceId, condition) +#define LOGW_IF_(instanceId, condition) PLOGW_IF_(instanceId, condition) +#define LOGE_IF_(instanceId, condition) PLOGE_IF_(instanceId, condition) +#define LOGF_IF_(instanceId, condition) PLOGF_IF_(instanceId, condition) +#define LOGN_IF_(instanceId, condition) PLOGN_IF_(instanceId, condition) +#endif diff --git a/include/plog/Logger.h b/include/plog/Logger.h new file mode 100644 index 000000000..0e116e4c6 --- /dev/null +++ b/include/plog/Logger.h @@ -0,0 +1,84 @@ +#pragma once +#include +#include +#include + +#ifdef PLOG_DEFAULT_INSTANCE // for backward compatibility +# define PLOG_DEFAULT_INSTANCE_ID PLOG_DEFAULT_INSTANCE +#endif + +#ifndef PLOG_DEFAULT_INSTANCE_ID +# define PLOG_DEFAULT_INSTANCE_ID 0 +#endif + +namespace plog +{ + template + class PLOG_LINKAGE Logger : public util::Singleton >, public IAppender + { + public: + Logger(Severity maxSeverity = none) : m_maxSeverity(maxSeverity) + { + } + + Logger& addAppender(IAppender* appender) + { + assert(appender != this); + m_appenders.push_back(appender); + return *this; + } + + Severity getMaxSeverity() const + { + return m_maxSeverity; + } + + void setMaxSeverity(Severity severity) + { + m_maxSeverity = severity; + } + + bool checkSeverity(Severity severity) const + { + return severity <= m_maxSeverity; + } + + virtual void write(const Record& record) PLOG_OVERRIDE + { + if (checkSeverity(record.getSeverity())) + { + *this += record; + } + } + + void operator+=(const Record& record) + { + for (std::vector::iterator it = m_appenders.begin(); it != m_appenders.end(); ++it) + { + (*it)->write(record); + } + } + + private: + Severity m_maxSeverity; +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable:4251) // needs to have dll-interface to be used by clients of class +#endif + std::vector m_appenders; +#ifdef _MSC_VER +# pragma warning(pop) +#endif + }; + + template + inline Logger* get() + { + return Logger::getInstance(); + } + + inline Logger* get() + { + return Logger::getInstance(); + } +} diff --git a/include/plog/Record.h b/include/plog/Record.h new file mode 100644 index 000000000..c919e3aa6 --- /dev/null +++ b/include/plog/Record.h @@ -0,0 +1,465 @@ +#pragma once +#include +#include +#include + +#ifdef __cplusplus_cli +#include // For PtrToStringChars +#endif + +namespace plog +{ + namespace detail + { +#if !defined(_MSC_VER) || _MSC_VER > 1400 // MSVC 2005 doesn't understand `enableIf`, so drop all `meta` + namespace meta + { + template + inline T& declval() + { +#ifdef __INTEL_COMPILER +# pragma warning(suppress: 327) // NULL reference is not allowed +#endif + return *reinterpret_cast(0); + } + + template + struct enableIf {}; + + template + struct enableIf { typedef T type; }; + + struct No { char a[1]; }; + struct Yes { char a[2]; }; + + template + struct isConvertible + { + // `+ sizeof(U*)` is required for GCC 4.5-4.7 + template + static typename enableIf(meta::declval())) + sizeof(U*)), Yes>::type test(int); + + template + static No test(...); + + enum { value = sizeof(test(0)) == sizeof(Yes) }; + }; + + template + struct isConvertibleToString : isConvertible {}; + +#if PLOG_ENABLE_WCHAR_INPUT + template + struct isConvertibleToWString : isConvertible {}; +#endif + + template + struct isContainer + { + template + static typename meta::enableIf().begin()) + sizeof(meta::declval().end() +#else + typename U::const_iterator +#endif + )), Yes>::type test(int); + + template + static No test(...); + + enum { value = sizeof(test(0)) == sizeof(Yes) }; + }; + + // Detects `std::filesystem::path` and `boost::filesystem::path`. They look like containers + // but we don't want to treat them as containers, so we use this detector to filter them out. + template + struct isFilesystemPath + { + template + static typename meta::enableIf().preferred_separator)), Yes>::type test(int); + + template + static No test(...); + + enum { value = sizeof(test(0)) == sizeof(Yes) }; + }; + } +#endif + + ////////////////////////////////////////////////////////////////////////// + // Stream output operators as free functions + +#if PLOG_ENABLE_WCHAR_INPUT + inline void operator<<(util::nostringstream& stream, const wchar_t* data) + { + data = data ? data : L"(null)"; + +# ifdef _WIN32 +# if PLOG_CHAR_IS_UTF8 + std::operator<<(stream, util::toNarrow(data, codePage::kUTF8)); +# else + std::operator<<(stream, data); +# endif +# else + std::operator<<(stream, util::toNarrow(data)); +# endif + } + + inline void operator<<(util::nostringstream& stream, wchar_t* data) + { + plog::detail::operator<<(stream, const_cast(data)); + } + + inline void operator<<(util::nostringstream& stream, const std::wstring& data) + { + plog::detail::operator<<(stream, data.c_str()); + } +#endif + + inline void operator<<(util::nostringstream& stream, const char* data) + { + data = data ? data : "(null)"; + +#if defined(_WIN32) && defined(__BORLANDC__) +# if PLOG_CHAR_IS_UTF8 + stream << data; +# else + stream << util::toWide(data); +# endif +#elif defined(_WIN32) +# if PLOG_CHAR_IS_UTF8 + std::operator<<(stream, data); +# else + std::operator<<(stream, util::toWide(data)); +# endif +#else + std::operator<<(stream, data); +#endif + } + + inline void operator<<(util::nostringstream& stream, char* data) + { + plog::detail::operator<<(stream, const_cast(data)); + } + + inline void operator<<(util::nostringstream& stream, const std::string& data) + { + plog::detail::operator<<(stream, data.c_str()); + } + +#ifdef __cpp_char8_t + inline void operator<<(util::nostringstream& stream, const char8_t* data) + { +# if PLOG_CHAR_IS_UTF8 + plog::detail::operator<<(stream, reinterpret_cast(data)); +# else + plog::detail::operator<<(stream, util::toWide(reinterpret_cast(data), codePage::kUTF8)); +# endif + } +#endif //__cpp_char8_t + + // Print `std::pair` + template + inline void operator<<(util::nostringstream& stream, const std::pair& data) + { + stream << data.first; + stream << ":"; + stream << data.second; + } + +#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ * 100 + __GNUC_MINOR__) >= 405 // skip for GCC < 4.5 due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=38600 +#if !defined(_MSC_VER) || _MSC_VER > 1400 // MSVC 2005 doesn't understand `enableIf`, so drop all `meta` + // Print data that can be casted to `std::string` + template + inline typename meta::enableIf::value, void>::type operator<<(util::nostringstream& stream, const T& data) + { + plog::detail::operator<<(stream, static_cast(data)); + } + +#if PLOG_ENABLE_WCHAR_INPUT + // Print data that can be casted to `std::wstring` + template + inline typename meta::enableIf::value, void>::type operator<<(util::nostringstream& stream, const T& data) + { + plog::detail::operator<<(stream, static_cast(data)); + } +#endif + + // Print std containers + template + inline typename meta::enableIf::value && + !meta::isConvertibleToString::value && +#if PLOG_ENABLE_WCHAR_INPUT + !meta::isConvertibleToWString::value && +#endif + !meta::isFilesystemPath::value, void>::type operator<<(util::nostringstream& stream, const T& data) + { + stream << "["; + + for (typename T::const_iterator it = data.begin(); it != data.end();) + { + stream << *it; + + if (++it == data.end()) + { + break; + } + + stream << ", "; + } + + stream << "]"; + } +#endif +#endif + +#ifdef __cplusplus_cli + // Print managed C++ `System::String^` + inline void operator<<(util::nostringstream& stream, System::String^ data) + { + cli::pin_ptr ptr = PtrToStringChars(data); + plog::detail::operator<<(stream, static_cast(ptr)); + } +#endif + +#if PLOG_ENABLE_WCHAR_INPUT && (!defined(_MSC_VER) || _MSC_VER > 1400) // MSVC 2005 doesn't understand `enableIf`, so drop all `meta` + namespace meta + { + template + struct valueType { enum { value = Value }; }; + + template + inline No operator<<(Stream&, const T&); + + template + struct isStreamable : valueType(), meta::declval())) != sizeof(No)> {}; + + template + struct isStreamable : valueType {}; + + template + struct isStreamable : valueType {}; + + template + struct isStreamable : valueType {}; + + // meta doesn't work well for deleted functions and C++20 has `operator<<(std::ostream&, const wchar_t*) = delete` so explicitly define it + template<> + struct isStreamable : valueType {}; + +# ifdef __cpp_char8_t + // meta doesn't work well for deleted functions and C++20 has `operator<<(std::ostream&, const char8_t*) = delete` so explicitly define it + template + struct isStreamable : valueType {}; + + template + struct isStreamable : valueType {}; +# endif //__cpp_char8_t + } + +# if PLOG_CHAR_IS_UTF8 + // Print types that can be streamed into `std::owstringstream` but not into `std::ostringstream` when we use UTF-8 on Windows + template + inline typename meta::enableIf::value && + !meta::isStreamable::value && + !meta::isConvertibleToWString::value, void>::type operator<<(std::ostringstream& stream, const T& data) + { + std::wostringstream ss; + ss << data; + stream << ss.str(); + } +# else + // Print types that can be streamed into `std::ostringstream` but not into `std::owstringstream` when we use `wchar_t` on Windows + template + inline typename meta::enableIf::value && + !meta::isStreamable::value && + !meta::isConvertibleToString::value, void>::type operator<<(std::wostringstream& stream, const T& data) + { + std::ostringstream ss; + ss << data; + stream << ss.str(); + } +# endif +#endif + } + + class Record + { + public: + Record(Severity severity, const char* func, size_t line, const char* file, const void* object, int instanceId) + : m_severity(severity), m_tid(util::gettid()), m_object(object), m_line(line), m_func(func), m_file(file), m_instanceId(instanceId) + { + util::ftime(&m_time); + } + + Record& ref() + { + return *this; + } + + ////////////////////////////////////////////////////////////////////////// + // Stream output operators + + Record& operator<<(char data) + { + char str[] = { data, 0 }; + return *this << str; + } + +#if PLOG_ENABLE_WCHAR_INPUT + Record& operator<<(wchar_t data) + { + wchar_t str[] = { data, 0 }; + return *this << str; + } +#endif + + Record& operator<<(util::nostream& (PLOG_CDECL *data)(util::nostream&)) + { + m_message << data; + return *this; + } + +#ifdef QT_VERSION + Record& operator<<(const QString& data) + { +# if PLOG_CHAR_IS_UTF8 + return *this << data.toStdString(); +# else + return *this << data.toStdWString(); +# endif + } + +# if QT_VERSION < 0x060000 + Record& operator<<(const QStringRef& data) + { + return *this << data.toString(); + } +# endif + +# ifdef QSTRINGVIEW_H + Record& operator<<(QStringView data) + { + return *this << data.toString(); + } +# endif +#endif + + template + Record& operator<<(const T& data) + { + using namespace plog::detail; + + m_message << data; + return *this; + } + +#ifndef __cplusplus_cli + Record& printf(const char* format, ...) + { + using namespace util; + + char* str = NULL; + va_list ap; + + va_start(ap, format); + int len = vasprintf(&str, format, ap); + static_cast(len); + va_end(ap); + + *this << str; + free(str); + + return *this; + } + +#ifdef _WIN32 + Record& printf(const wchar_t* format, ...) + { + using namespace util; + + wchar_t* str = NULL; + va_list ap; + + va_start(ap, format); + int len = vaswprintf(&str, format, ap); + static_cast(len); + va_end(ap); + + *this << str; + free(str); + + return *this; + } +#endif +#endif //__cplusplus_cli + + ////////////////////////////////////////////////////////////////////////// + // Getters + + virtual const util::Time& getTime() const + { + return m_time; + } + + virtual Severity getSeverity() const + { + return m_severity; + } + + virtual unsigned int getTid() const + { + return m_tid; + } + + virtual const void* getObject() const + { + return m_object; + } + + virtual size_t getLine() const + { + return m_line; + } + + virtual const util::nchar* getMessage() const + { + m_messageStr = m_message.str(); + return m_messageStr.c_str(); + } + + virtual const char* getFunc() const + { + m_funcStr = util::processFuncName(m_func); + return m_funcStr.c_str(); + } + + virtual const char* getFile() const + { + return m_file; + } + + virtual ~Record() // virtual destructor to satisfy -Wnon-virtual-dtor warning + { + } + + virtual int getInstanceId() const + { + return m_instanceId; + } + + private: + util::Time m_time; + const Severity m_severity; + const unsigned int m_tid; + const void* const m_object; + const size_t m_line; + util::nostringstream m_message; + const char* const m_func; + const char* const m_file; + const int m_instanceId; + mutable std::string m_funcStr; + mutable util::nstring m_messageStr; + }; +} diff --git a/include/plog/Severity.h b/include/plog/Severity.h new file mode 100644 index 000000000..446768e8f --- /dev/null +++ b/include/plog/Severity.h @@ -0,0 +1,61 @@ +#pragma once +#include + +namespace plog +{ + enum Severity + { + none = 0, + fatal = 1, + error = 2, + warning = 3, + info = 4, + debug = 5, + verbose = 6 + }; + +#ifdef _MSC_VER +# pragma warning(suppress: 26812) // Prefer 'enum class' over 'enum' +#endif + inline const char* severityToString(Severity severity) + { + switch (severity) + { + case fatal: + return "FATAL"; + case error: + return "ERROR"; + case warning: + return "WARN"; + case info: + return "INFO"; + case debug: + return "DEBUG"; + case verbose: + return "VERB"; + default: + return "NONE"; + } + } + + inline Severity severityFromString(const char* str) + { + switch (std::toupper(str[0])) + { + case 'F': + return fatal; + case 'E': + return error; + case 'W': + return warning; + case 'I': + return info; + case 'D': + return debug; + case 'V': + return verbose; + default: + return none; + } + } +} diff --git a/include/plog/Util.h b/include/plog/Util.h new file mode 100644 index 000000000..ec319a10c --- /dev/null +++ b/include/plog/Util.h @@ -0,0 +1,635 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#ifndef PLOG_ENABLE_WCHAR_INPUT +# ifdef _WIN32 +# define PLOG_ENABLE_WCHAR_INPUT 1 +# else +# define PLOG_ENABLE_WCHAR_INPUT 0 +# endif +#endif + +////////////////////////////////////////////////////////////////////////// +// PLOG_CHAR_IS_UTF8 specifies character encoding of `char` type. On *nix +// systems it's set to UTF-8 while on Windows in can be ANSI or UTF-8. It +// automatically detects `/utf-8` command line option in MSVC. Also it can +// be set manually if required. +// This option allows to support http://utf8everywhere.org approach. + +#ifndef PLOG_CHAR_IS_UTF8 +# if defined(_WIN32) && !defined(_UTF8) +# define PLOG_CHAR_IS_UTF8 0 +# else +# define PLOG_CHAR_IS_UTF8 1 +# endif +#endif + +#ifdef _WIN32 +# if defined(PLOG_EXPORT) +# define PLOG_LINKAGE __declspec(dllexport) +# elif defined(PLOG_IMPORT) +# define PLOG_LINKAGE __declspec(dllimport) +# endif +# if defined(PLOG_GLOBAL) +# error "PLOG_GLOBAL isn't supported on Windows" +# endif +#else +# if defined(PLOG_GLOBAL) +# define PLOG_LINKAGE __attribute__ ((visibility ("default"))) +# elif defined(PLOG_LOCAL) +# define PLOG_LINKAGE __attribute__ ((visibility ("hidden"))) +# define PLOG_LINKAGE_HIDDEN PLOG_LINKAGE +# endif +# if defined(PLOG_EXPORT) || defined(PLOG_IMPORT) +# error "PLOG_EXPORT/PLOG_IMPORT is supported only on Windows" +# endif +#endif + +#ifndef PLOG_LINKAGE +# define PLOG_LINKAGE +#endif + +#ifndef PLOG_LINKAGE_HIDDEN +# define PLOG_LINKAGE_HIDDEN +#endif + +#ifdef _WIN32 +# include +# include +# include +# include +# include +#else +# include +# include +# if defined(__linux__) || defined(__FreeBSD__) +# include +# elif defined(__rtems__) +# include +# endif +# if defined(_POSIX_THREADS) +# include +# endif +# if PLOG_ENABLE_WCHAR_INPUT +# include +# endif +#endif + +#ifdef __FREERTOS__ // There is no standard way to know if the code is compiled for FreeRTOS. We expect __FREERTOS__ macro to be defined in such case. +# include +# include +# include +#endif + +#if PLOG_CHAR_IS_UTF8 +# define PLOG_NSTR(x) x +#else +# define _PLOG_NSTR(x) L##x +# define PLOG_NSTR(x) _PLOG_NSTR(x) +#endif + +#ifdef _WIN32 +# define PLOG_CDECL __cdecl +#else +# define PLOG_CDECL +#endif + +#if __cplusplus >= 201103L || defined(_MSC_VER) && _MSC_VER >= 1700 +# define PLOG_OVERRIDE override +#else +# define PLOG_OVERRIDE +#endif + +namespace plog +{ + namespace util + { +#if PLOG_CHAR_IS_UTF8 + typedef std::string nstring; + typedef std::ostringstream nostringstream; + typedef std::istringstream nistringstream; + typedef std::ostream nostream; + typedef char nchar; +#else + typedef std::wstring nstring; + typedef std::wostringstream nostringstream; + typedef std::wistringstream nistringstream; + typedef std::wostream nostream; + typedef wchar_t nchar; +#endif + + inline void localtime_s(struct tm* t, const time_t* time) + { +#if defined(_WIN32) && defined(__BORLANDC__) + ::localtime_s(time, t); +#elif defined(_WIN32) && defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) + *t = *::localtime(time); +#elif defined(_WIN32) + ::localtime_s(t, time); +#else + ::localtime_r(time, t); +#endif + } + + inline void gmtime_s(struct tm* t, const time_t* time) + { +#if defined(_WIN32) && defined(__BORLANDC__) + ::gmtime_s(time, t); +#elif defined(_WIN32) && defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) + *t = *::gmtime(time); +#elif defined(_WIN32) + ::gmtime_s(t, time); +#else + ::gmtime_r(time, t); +#endif + } + +#ifdef _WIN32 + typedef timeb Time; + + inline void ftime(Time* t) + { + ::ftime(t); + } +#else + struct Time + { + time_t time; + unsigned short millitm; + }; + + inline void ftime(Time* t) + { + timeval tv; + ::gettimeofday(&tv, NULL); + + t->time = tv.tv_sec; + t->millitm = static_cast(tv.tv_usec / 1000); + } +#endif + + inline unsigned int gettid() + { +#if defined(__FREERTOS__) && defined(INCLUDE_xTaskGetCurrentTaskHandle) + return static_cast(reinterpret_cast(xTaskGetCurrentTaskHandle())); +#elif defined(_WIN32) + return GetCurrentThreadId(); +#elif defined(__linux__) + return static_cast(::syscall(__NR_gettid)); +#elif defined(__FreeBSD__) + long tid; + syscall(SYS_thr_self, &tid); + return static_cast(tid); +#elif defined(__rtems__) + return rtems_task_self(); +#elif defined(__APPLE__) + uint64_t tid64; + pthread_threadid_np(NULL, &tid64); + return static_cast(tid64); +#else + return 0; +#endif + } + +#ifndef _GNU_SOURCE + inline int vasprintf(char** strp, const char* format, va_list ap) + { + va_list ap_copy; +#if defined(_MSC_VER) && _MSC_VER <= 1600 + ap_copy = ap; // there is no va_copy on Visual Studio 2010 +#else + va_copy(ap_copy, ap); +#endif +#ifndef __STDC_SECURE_LIB__ + int charCount = vsnprintf(NULL, 0, format, ap_copy); +#else + int charCount = _vscprintf(format, ap_copy); +#endif + va_end(ap_copy); + if (charCount < 0) + { + return -1; + } + + size_t bufferCharCount = static_cast(charCount) + 1; + + char* str = static_cast(malloc(bufferCharCount)); + if (!str) + { + return -1; + } + +#ifndef __STDC_SECURE_LIB__ + int retval = vsnprintf(str, bufferCharCount, format, ap); +#else + int retval = vsnprintf_s(str, bufferCharCount, static_cast(-1), format, ap); +#endif + if (retval < 0) + { + free(str); + return -1; + } + + *strp = str; + return retval; + } +#endif + +#ifdef _WIN32 + inline int vaswprintf(wchar_t** strp, const wchar_t* format, va_list ap) + { +#if defined(__BORLANDC__) + int charCount = 0x1000; // there is no _vscwprintf on Borland/Embarcadero +#else + int charCount = _vscwprintf(format, ap); + if (charCount < 0) + { + return -1; + } +#endif + + size_t bufferCharCount = static_cast(charCount) + 1; + + wchar_t* str = static_cast(malloc(bufferCharCount * sizeof(wchar_t))); + if (!str) + { + return -1; + } + +#if defined(__BORLANDC__) + int retval = vsnwprintf_s(str, bufferCharCount, format, ap); +#elif defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) + int retval = _vsnwprintf(str, bufferCharCount, format, ap); +#else + int retval = _vsnwprintf_s(str, bufferCharCount, charCount, format, ap); +#endif + if (retval < 0) + { + free(str); + return -1; + } + + *strp = str; + return retval; + } +#endif + +#ifdef _WIN32 + inline std::wstring toWide(const char* str, UINT cp = codePage::kChar) + { + size_t len = ::strlen(str); + std::wstring wstr(len, 0); + + if (!wstr.empty()) + { + int wlen = MultiByteToWideChar(cp, 0, str, static_cast(len), &wstr[0], static_cast(wstr.size())); + wstr.resize(wlen); + } + + return wstr; + } + + inline std::wstring toWide(const std::string& str, UINT cp = codePage::kChar) + { + return toWide(str.c_str(), cp); + } + + inline const std::wstring& toWide(const std::wstring& str) // do nothing for already wide string + { + return str; + } + + inline std::string toNarrow(const std::wstring& wstr, long page) + { + int len = WideCharToMultiByte(page, 0, wstr.c_str(), static_cast(wstr.size()), 0, 0, 0, 0); + std::string str(len, 0); + + if (!str.empty()) + { + WideCharToMultiByte(page, 0, wstr.c_str(), static_cast(wstr.size()), &str[0], len, 0, 0); + } + + return str; + } +#elif PLOG_ENABLE_WCHAR_INPUT + inline std::string toNarrow(const wchar_t* wstr) + { + size_t wlen = ::wcslen(wstr); + std::string str(wlen * sizeof(wchar_t), 0); + + if (!str.empty()) + { + const char* in = reinterpret_cast(&wstr[0]); + char* out = &str[0]; + size_t inBytes = wlen * sizeof(wchar_t); + size_t outBytes = str.size(); + + iconv_t cd = ::iconv_open("UTF-8", "WCHAR_T"); + ::iconv(cd, const_cast(&in), &inBytes, &out, &outBytes); + ::iconv_close(cd); + + str.resize(str.size() - outBytes); + } + + return str; + } +#endif + + inline std::string processFuncName(const char* func) + { +#if (defined(_WIN32) && !defined(__MINGW32__)) || defined(__OBJC__) + return std::string(func); +#else + const char* funcBegin = func; + const char* funcEnd = ::strchr(funcBegin, '('); + int foundTemplate = 0; + + if (!funcEnd) + { + return std::string(func); + } + + for (const char* i = funcEnd - 1; i >= funcBegin; --i) // search backwards for the first space char + { + if (*i == '>') + { + foundTemplate++; + } + else if (*i == '<') + { + foundTemplate--; + } + else if (*i == ' ' && foundTemplate == 0) + { + funcBegin = i + 1; + break; + } + } + + return std::string(funcBegin, funcEnd); +#endif + } + + inline const nchar* findExtensionDot(const nchar* fileName) + { +#if PLOG_CHAR_IS_UTF8 + return std::strrchr(fileName, '.'); +#else + return std::wcsrchr(fileName, L'.'); +#endif + } + + inline void splitFileName(const nchar* fileName, nstring& fileNameNoExt, nstring& fileExt) + { + const nchar* dot = findExtensionDot(fileName); + + if (dot) + { + fileNameNoExt.assign(fileName, dot); + fileExt.assign(dot + 1); + } + else + { + fileNameNoExt.assign(fileName); + fileExt.clear(); + } + } + + class PLOG_LINKAGE NonCopyable + { + protected: + NonCopyable() + { + } + + private: + NonCopyable(const NonCopyable&); + NonCopyable& operator=(const NonCopyable&); + }; + + class PLOG_LINKAGE_HIDDEN File : NonCopyable + { + public: + File() : m_file(-1) + { + } + + ~File() + { + close(); + } + + size_t open(const nstring& fileName) + { +#if defined(_WIN32) && (defined(__BORLANDC__) || defined(__MINGW32__)) + m_file = ::_wsopen(toWide(fileName).c_str(), _O_CREAT | _O_WRONLY | _O_BINARY | _O_NOINHERIT, SH_DENYWR, _S_IREAD | _S_IWRITE); +#elif defined(_WIN32) + ::_wsopen_s(&m_file, toWide(fileName).c_str(), _O_CREAT | _O_WRONLY | _O_BINARY | _O_NOINHERIT, _SH_DENYWR, _S_IREAD | _S_IWRITE); +#elif defined(O_CLOEXEC) + m_file = ::open(fileName.c_str(), O_CREAT | O_APPEND | O_WRONLY | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); +#else + m_file = ::open(fileName.c_str(), O_CREAT | O_APPEND | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); +#endif + return seek(0, SEEK_END); + } + + size_t write(const void* buf, size_t count) + { + return m_file != -1 ? static_cast( +#ifdef _WIN32 + ::_write(m_file, buf, static_cast(count)) +#else + ::write(m_file, buf, count) +#endif + ) : static_cast(-1); + } + + template + size_t write(const std::basic_string& str) + { + return write(str.data(), str.size() * sizeof(CharType)); + } + + size_t seek(size_t offset, int whence) + { + return m_file != -1 ? static_cast( +#if defined(_WIN32) && (defined(__BORLANDC__) || defined(__MINGW32__)) + ::_lseek(m_file, static_cast(offset), whence) +#elif defined(_WIN32) + ::_lseeki64(m_file, static_cast(offset), whence) +#else + ::lseek(m_file, static_cast(offset), whence) +#endif + ) : static_cast(-1); + } + + void close() + { + if (m_file != -1) + { +#ifdef _WIN32 + ::_close(m_file); +#else + ::close(m_file); +#endif + m_file = -1; + } + } + + static int unlink(const nstring& fileName) + { +#ifdef _WIN32 + return ::_wunlink(toWide(fileName).c_str()); +#else + return ::unlink(fileName.c_str()); +#endif + } + + static int rename(const nstring& oldFilename, const nstring& newFilename) + { +#ifdef _WIN32 + return MoveFileW(toWide(oldFilename).c_str(), toWide(newFilename).c_str()); +#else + return ::rename(oldFilename.c_str(), newFilename.c_str()); +#endif + } + + private: + int m_file; + }; + + class PLOG_LINKAGE_HIDDEN Mutex : NonCopyable + { + public: + Mutex() + { +#ifdef __FREERTOS__ + m_sync = xSemaphoreCreateBinary(); + xSemaphoreGive(m_sync); +#elif defined(_WIN32) + InitializeCriticalSection(&m_sync); +#elif defined(__rtems__) + rtems_semaphore_create(0, 1, + RTEMS_PRIORITY | + RTEMS_BINARY_SEMAPHORE | + RTEMS_INHERIT_PRIORITY, 1, &m_sync); +#elif defined(_POSIX_THREADS) + ::pthread_mutex_init(&m_sync, 0); +#endif + } + + ~Mutex() + { +#ifdef __FREERTOS__ + vSemaphoreDelete(m_sync); +#elif defined(_WIN32) + DeleteCriticalSection(&m_sync); +#elif defined(__rtems__) + rtems_semaphore_delete(m_sync); +#elif defined(_POSIX_THREADS) + ::pthread_mutex_destroy(&m_sync); +#endif + } + + friend class MutexLock; + + private: + void lock() + { +#ifdef __FREERTOS__ + xSemaphoreTake(m_sync, portMAX_DELAY); +#elif defined(_WIN32) + EnterCriticalSection(&m_sync); +#elif defined(__rtems__) + rtems_semaphore_obtain(m_sync, RTEMS_WAIT, RTEMS_NO_TIMEOUT); +#elif defined(_POSIX_THREADS) + ::pthread_mutex_lock(&m_sync); +#endif + } + + void unlock() + { +#ifdef __FREERTOS__ + xSemaphoreGive(m_sync); +#elif defined(_WIN32) + LeaveCriticalSection(&m_sync); +#elif defined(__rtems__) + rtems_semaphore_release(m_sync); +#elif defined(_POSIX_THREADS) + ::pthread_mutex_unlock(&m_sync); +#endif + } + + private: +#ifdef __FREERTOS__ + SemaphoreHandle_t m_sync; +#elif defined(_WIN32) + CRITICAL_SECTION m_sync; +#elif defined(__rtems__) + rtems_id m_sync; +#elif defined(_POSIX_THREADS) + pthread_mutex_t m_sync; +#endif + }; + + class PLOG_LINKAGE_HIDDEN MutexLock : NonCopyable + { + public: + MutexLock(Mutex& mutex) : m_mutex(mutex) + { + m_mutex.lock(); + } + + ~MutexLock() + { + m_mutex.unlock(); + } + + private: + Mutex& m_mutex; + }; + + template +#ifdef _WIN32 + class Singleton : NonCopyable +#else + class PLOG_LINKAGE Singleton : NonCopyable +#endif + { + public: +#if (defined(__clang__) || defined(__GNUC__) && __GNUC__ >= 8) && !defined(__BORLANDC__) + // This constructor is called before the `T` object is fully constructed, and + // pointers are not dereferenced anyway, so UBSan shouldn't check vptrs. + __attribute__((no_sanitize("vptr"))) +#endif + Singleton() + { + assert(!m_instance); + m_instance = static_cast(this); + } + + ~Singleton() + { + assert(m_instance); + m_instance = 0; + } + + static T* getInstance() + { + return m_instance; + } + + private: + static T* m_instance; + }; + + template + T* Singleton::m_instance = NULL; + } +} diff --git a/include/plog/WinApi.h b/include/plog/WinApi.h new file mode 100644 index 000000000..ccf44af0a --- /dev/null +++ b/include/plog/WinApi.h @@ -0,0 +1,175 @@ +#pragma once + +#ifdef _WIN32 + +// These windows structs must be in a global namespace +struct HKEY__; +struct _SECURITY_ATTRIBUTES; +struct _CONSOLE_SCREEN_BUFFER_INFO; +struct _RTL_CRITICAL_SECTION; + +namespace plog +{ + typedef unsigned long DWORD; + typedef unsigned short WORD; + typedef unsigned char BYTE; + typedef unsigned int UINT; + typedef int BOOL; + typedef long LSTATUS; + typedef char* LPSTR; + typedef wchar_t* LPWSTR; + typedef const char* LPCSTR; + typedef const wchar_t* LPCWSTR; + typedef void* HANDLE; + typedef HKEY__* HKEY; + typedef size_t ULONG_PTR; + + struct CRITICAL_SECTION + { + void* DebugInfo; + long LockCount; + long RecursionCount; + HANDLE OwningThread; + HANDLE LockSemaphore; + ULONG_PTR SpinCount; + }; + + struct COORD + { + short X; + short Y; + }; + + struct SMALL_RECT + { + short Left; + short Top; + short Right; + short Bottom; + }; + + struct CONSOLE_SCREEN_BUFFER_INFO + { + COORD dwSize; + COORD dwCursorPosition; + WORD wAttributes; + SMALL_RECT srWindow; + COORD dwMaximumWindowSize; + }; + + namespace codePage + { + const UINT kActive = 0; + const UINT kUTF8 = 65001; +#if PLOG_CHAR_IS_UTF8 + const UINT kChar = kUTF8; +#else + const UINT kChar = kActive; +#endif + } + + namespace eventLog + { + const WORD kErrorType = 0x0001; + const WORD kWarningType = 0x0002; + const WORD kInformationType = 0x0004; + } + + namespace hkey + { + const HKEY kLocalMachine = reinterpret_cast(static_cast(0x80000002)); + } + + namespace regSam + { + const DWORD kQueryValue = 0x0001; + const DWORD kSetValue = 0x0002; + } + + namespace regType + { + const DWORD kExpandSz = 2; + const DWORD kDword = 4; + } + + namespace stdHandle + { + const DWORD kOutput = static_cast(-11); + const DWORD kErrorOutput = static_cast(-12); + } + + namespace foreground + { + const WORD kBlue = 0x0001; + const WORD kGreen = 0x0002; + const WORD kRed = 0x0004; + const WORD kIntensity = 0x0008; + } + + namespace background + { + const WORD kBlue = 0x0010; + const WORD kGreen = 0x0020; + const WORD kRed = 0x0040; + const WORD kIntensity = 0x0080; + } + + extern "C" + { + __declspec(dllimport) int __stdcall MultiByteToWideChar(UINT CodePage, DWORD dwFlags, LPCSTR lpMultiByteStr, int cbMultiByte, LPWSTR lpWideCharStr, int cchWideChar); + __declspec(dllimport) int __stdcall WideCharToMultiByte(UINT CodePage, DWORD dwFlags, LPCWSTR lpWideCharStr, int cchWideChar, LPSTR lpMultiByteStr, int cbMultiByte, const char* lpDefaultChar, BOOL* lpUsedDefaultChar); + + __declspec(dllimport) DWORD __stdcall GetCurrentThreadId(); + + __declspec(dllimport) BOOL __stdcall MoveFileW(LPCWSTR lpExistingFileName, LPCWSTR lpNewFileName); + + __declspec(dllimport) void __stdcall InitializeCriticalSection(_RTL_CRITICAL_SECTION* lpCriticalSection); + __declspec(dllimport) void __stdcall EnterCriticalSection(_RTL_CRITICAL_SECTION* lpCriticalSection); + __declspec(dllimport) void __stdcall LeaveCriticalSection(_RTL_CRITICAL_SECTION* lpCriticalSection); + __declspec(dllimport) void __stdcall DeleteCriticalSection(_RTL_CRITICAL_SECTION* lpCriticalSection); + + __declspec(dllimport) HANDLE __stdcall RegisterEventSourceW(LPCWSTR lpUNCServerName, LPCWSTR lpSourceName); + __declspec(dllimport) BOOL __stdcall DeregisterEventSource(HANDLE hEventLog); + __declspec(dllimport) BOOL __stdcall ReportEventW(HANDLE hEventLog, WORD wType, WORD wCategory, DWORD dwEventID, void* lpUserSid, WORD wNumStrings, DWORD dwDataSize, LPCWSTR* lpStrings, void* lpRawData); + + __declspec(dllimport) LSTATUS __stdcall RegCreateKeyExW(HKEY hKey, LPCWSTR lpSubKey, DWORD Reserved, LPWSTR lpClass, DWORD dwOptions, DWORD samDesired, _SECURITY_ATTRIBUTES* lpSecurityAttributes, HKEY* phkResult, DWORD* lpdwDisposition); + __declspec(dllimport) LSTATUS __stdcall RegSetValueExW(HKEY hKey, LPCWSTR lpValueName, DWORD Reserved, DWORD dwType, const BYTE* lpData, DWORD cbData); + __declspec(dllimport) LSTATUS __stdcall RegCloseKey(HKEY hKey); + __declspec(dllimport) LSTATUS __stdcall RegOpenKeyExW(HKEY hKey, LPCWSTR lpSubKey, DWORD ulOptions, DWORD samDesired, HKEY* phkResult); + __declspec(dllimport) LSTATUS __stdcall RegDeleteKeyW(HKEY hKey, LPCWSTR lpSubKey); + + __declspec(dllimport) HANDLE __stdcall GetStdHandle(DWORD nStdHandle); + + __declspec(dllimport) BOOL __stdcall WriteConsoleW(HANDLE hConsoleOutput, const void* lpBuffer, DWORD nNumberOfCharsToWrite, DWORD* lpNumberOfCharsWritten, void* lpReserved); + __declspec(dllimport) BOOL __stdcall GetConsoleScreenBufferInfo(HANDLE hConsoleOutput, _CONSOLE_SCREEN_BUFFER_INFO* lpConsoleScreenBufferInfo); + __declspec(dllimport) BOOL __stdcall SetConsoleTextAttribute(HANDLE hConsoleOutput, WORD wAttributes); + + __declspec(dllimport) void __stdcall OutputDebugStringW(LPCWSTR lpOutputString); + } + + inline void InitializeCriticalSection(CRITICAL_SECTION* criticalSection) + { + plog::InitializeCriticalSection(reinterpret_cast<_RTL_CRITICAL_SECTION*>(criticalSection)); + } + + inline void EnterCriticalSection(CRITICAL_SECTION* criticalSection) + { + plog::EnterCriticalSection(reinterpret_cast<_RTL_CRITICAL_SECTION*>(criticalSection)); + } + + inline void LeaveCriticalSection(CRITICAL_SECTION* criticalSection) + { + plog::LeaveCriticalSection(reinterpret_cast<_RTL_CRITICAL_SECTION*>(criticalSection)); + } + + inline void DeleteCriticalSection(CRITICAL_SECTION* criticalSection) + { + plog::DeleteCriticalSection(reinterpret_cast<_RTL_CRITICAL_SECTION*>(criticalSection)); + } + + inline BOOL GetConsoleScreenBufferInfo(HANDLE consoleOutput, CONSOLE_SCREEN_BUFFER_INFO* consoleScreenBufferInfo) + { + return plog::GetConsoleScreenBufferInfo(consoleOutput, reinterpret_cast<_CONSOLE_SCREEN_BUFFER_INFO*>(consoleScreenBufferInfo)); + } +} +#endif // _WIN32 diff --git a/include/toml11/LICENSE b/include/toml11/LICENSE new file mode 100644 index 000000000..f55c511d6 --- /dev/null +++ b/include/toml11/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Toru Niina + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/src/global/utils/toml.h b/include/toml11/toml.hpp similarity index 99% rename from src/global/utils/toml.h rename to include/toml11/toml.hpp index 5d9981e9f..9c1405213 100644 --- a/src/global/utils/toml.h +++ b/include/toml11/toml.hpp @@ -16456,12 +16456,12 @@ namespace toml { inline namespace literals { inline namespace toml_literals { - ::toml::value operator"" _toml(const char* str, std::size_t len); + ::toml::value operator""_toml(const char* str, std::size_t len); #if defined(TOML11_HAS_CHAR8_T) // value of u8"" literal has been changed from char to char8_t and char8_t // is NOT compatible to char - ::toml::value operator"" _toml(const char8_t* str, std::size_t len); + ::toml::value operator""_toml(const char8_t* str, std::size_t len); #endif } // namespace toml_literals @@ -16566,7 +16566,7 @@ namespace toml { inline namespace literals { inline namespace toml_literals { - TOML11_INLINE ::toml::value operator"" _toml(const char* str, + TOML11_INLINE ::toml::value operator""_toml(const char* str, std::size_t len) { if (len == 0) { return ::toml::value {}; @@ -16596,7 +16596,7 @@ namespace toml { #if defined(TOML11_HAS_CHAR8_T) // value of u8"" literal has been changed from char to char8_t and char8_t // is NOT compatible to char - TOML11_INLINE ::toml::value operator"" _toml(const char8_t* str, + TOML11_INLINE ::toml::value operator""_toml(const char8_t* str, std::size_t len) { if (len == 0) { return ::toml::value {}; diff --git a/input.example.toml b/input.example.toml index bffb1d71a..f890143a6 100644 --- a/input.example.toml +++ b/input.example.toml @@ -107,7 +107,7 @@ [grid.boundaries.match] # Size of the matching layer in each direction for fields in physical (code) units - # @type: float | array> + # @type: float | array> # @default: 1% of the domain size (in shortest dimension) # @note: In spherical, this is the size of the layer in `r` from the outer wall # @example: `ds = 1.5` (will set the same for all directions) @@ -195,6 +195,83 @@ # @from: `scales.larmor0` # @value: `1 / larmor0` +[radiation] + [radiation.drag] + [radiation.drag.synchrotron] + # Radiation reaction limit gamma-factor for synchrotron + # @type: float [> 0.0] + # @default: 1.0 + # @note: [required] if one of the species has `radiative_drag = "synchrotron"` + gamma_rad = "" + + [radiation.drag.compton] + # Radiation reaction limit gamma-factor for Compton drag + # @type: float [> 0.0] + # @default: 1.0 + # @note: [required] if one of the species has `radiative_drag = "compton"` + gamma_rad = "" + + [radiation.emission] + [radiation.emission.synchrotron] + # Gamma-factor of a particle emitting synchrotron photons at energy `m0 c^2` in fiducial magnetic field `B0` + # @type: float [> 1.0] + # @default: 10.0 + gamma_qed = "" + # Minimum photon energy for synchrotron emission (units of `m0 c^2`) + # @type: float [> 0.0] + # @default: 1e-4 + photon_energy_min = "" + # Weights for the emitted synchrotron photons + # @type: float [> 0.0] + # @default: 1.0 + photon_weight = "" + # Index of species for the emitted photon + # @type: ushort [> 0] + # @required + photon_species = "" + + # @inferred: + # - nominal_probability + # @brief: Nominal probability of the emission for a particle with `gamma * beta = 1`, charge-to-mass = `q0 / m0` + # @type: float + # @from: `.gamma_qed`, `.photon_weight`, `...drag.synchrotron.gamma_rad`, `scales.omegaB0`, `algorithms.timestep.dt` + # @value: `0.1 * omegaB0 * dt * (gamma_qed / gamma_rad)^2 / photon_weight` + # - nominal_photon_energy + # @brief: Nominal energy of the emitted photon for a particle with `gamma * beta = 1`, mass = `m0` + # @type: float + # @from: `.gamma_qed` + # @value: `(1 / gamma_qed)^2` + + [radiation.emission.compton] + # Gamma-factor of a particle emitting inverse Compton photons at energy `m0 c^2` in fiducial magnetic field `B0` + # @type: float [> 1.0] + # @default: 10.0 + gamma_qed = "" + # Minimum photon energy for inverse Compton emission (units of `m0 c^2`) + # @type: float [> 0.0] + # @default: 1e-4 + photon_energy_min = "" + # Weights for the emitted inverse Compton photons + # @type: float [> 0.0] + # @default: 1.0 + photon_weight = "" + # Index of species for the emitted photon + # @type: ushort [> 0] + # @required + photon_species = "" + + # @inferred: + # - nominal_probability + # @brief: Nominal probability of the emission for a particle with `gamma * beta = 1`, charge-to-mass = `q0 / m0` + # @type: float + # @from: `.gamma_qed`, `.photon_weight`, `...drag.compton.gamma_rad`, `scales.omegaB0`, `algorithms.timestep.dt` + # @value: `0.1 * omegaB0 * dt * (gamma_qed / gamma_rad)^2 / photon_weight` + # - nominal_photon_energy + # @brief: Nominal energy of the emitted photon for a particle with `gamma * beta = 1`, mass = `m0` + # @type: float + # @from: `.gamma_qed` + # @value: `(1 / gamma_qed)^2` + [algorithms] # Number of current smoothing passes # @type: ushort [>= 0] @@ -224,10 +301,12 @@ # @type: bool # @default: true enable = "" - # Order of the particle shape function - # @type: int - # @default: 1 - order = "" + + # @inferred: + # - order + # @brief: order of the particle shape function + # @from: compile-time definition `shape_order` + # @type: ushort [0 -> 10] [algorithms.gr] # Stepsize for numerical differentiation in GR pusher @@ -250,13 +329,6 @@ # @note: When `larmor_max` == 0, the limit is disabled larmor_max = "" - [algorithms.synchrotron] - # Radiation reaction limit gamma-factor for synchrotron - # @type: float [> 0.0] - # @default: 1.0 - # @note: [required] if one of the species has `cooling = "synchrotron"` - gamma_rad = "" - # Stencil coefficients for the field solver [notation as in Blinne+ (2018)] # @note: Standard Yee solver: `delta_i = beta_ij = 0.0` [algorithms.fieldsolver] @@ -318,11 +390,16 @@ # @type: bool # @default: false use_weights = "" - # Timesteps between particle re-sorting (removing dead particles) + # Timesteps between particle re-sorting by tags (removing dead particles) # @type: uint # @default: 100 # @note: Set to 0 to disable re-sorting clear_interval = "" + # Timesteps between spatial sorting of particles (for better cache performance) + # @type: uint + # @default: 0 + # @note: Set to 0 to disable spatial sorting + spatial_sorting_interval = "" # @inferred: # - nspec @@ -370,8 +447,29 @@ # Radiation reaction to use for the species # @type: string # @default: "None" - # @enum: "None", "Synchrotron" - cooling = "" + # @enum: "None", "Synchrotron", "Compton" + # @note: Can also be coma-separated combination, e.g., "Synchrotron,Compton" + # @note: Relevant radiation.drag parameters should also be provided + radiative_drag = "" + # Particle emission policy for the species + # @type: string + # @default: "None" + # @enum: "None", "Synchrotron", "Compton" + # @note: Only one emission mechanism allowed + # @note: Appropriate radiation drag flag will be applied automatically (unless explicitly set to "None") + emission = "" + # Timesteps between spatial sorting of particles for given species + # @type: uint + # @default: 0 + # @note: Set to 0 to disable spatial sorting + # @note: Overrides `particles.spatial_sorting_interval` for the given species + spatial_sorting_interval = "" + # Timesteps between particle re-sorting by tags (removing dead particles) + # @type: uint + # @default: 100 + # @note: Set to 0 to disable re-sorting + # @note: Overrides `particles.clear_interval` for the given species + clear_interval = "" # Parameters for specific problem generators and setups [setup] @@ -553,7 +651,7 @@ # @note: 0 = disable checkpointing # @note: -1 = keep all checkpoints keep = "" - # Write a checkpoint once after a fixed walltime + # Write a checkpoint once after a fixed walltime # @type: string # @default: "00:00:00" # @note: The format is "HH:MM:SS" diff --git a/minimal/CMakeLists.txt b/minimal/CMakeLists.txt index b21dd0fec..c77775223 100644 --- a/minimal/CMakeLists.txt +++ b/minimal/CMakeLists.txt @@ -83,13 +83,16 @@ if("KOKKOS" IN_LIST MODES) endif() if("ADIOS2_NOMPI" IN_LIST MODES) - set(libs stdc++fs) + set(libs "") + if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "AppleClang") + list(APPEND libs stdc++fs) + endif() set(exec adios2-nompi.xc) set(src ${CMAKE_CURRENT_SOURCE_DIR}/adios2.cpp) find_kokkos() find_adios2() - list(APPEND libs Kokkos::kokkos adios2::cxx11) + list(APPEND libs Kokkos::kokkos adios2::cxx) add_executable(${exec} ${src}) @@ -97,14 +100,17 @@ if("ADIOS2_NOMPI" IN_LIST MODES) endif() if("ADIOS2_MPI" IN_LIST MODES) - set(libs stdc++fs) + set(libs "") + if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "AppleClang") + list(APPEND libs stdc++fs) + endif() set(exec adios2-mpi.xc) set(src ${CMAKE_CURRENT_SOURCE_DIR}/adios2.cpp) find_package(MPI REQUIRED) find_kokkos() find_adios2() - list(APPEND libs MPI::MPI_CXX Kokkos::kokkos adios2::cxx11_mpi) + list(APPEND libs MPI::MPI_CXX Kokkos::kokkos adios2::cxx_mpi) add_executable(${exec} ${src}) diff --git a/minimal/adios2.cpp b/minimal/adios2.cpp index cd4ca3d6f..c89bf383b 100644 --- a/minimal/adios2.cpp +++ b/minimal/adios2.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #if defined(MPI_ENABLED) #include diff --git a/pgens/accretion/pgen.hpp b/pgens/accretion/pgen.hpp index 42690f041..d79408588 100644 --- a/pgens/accretion/pgen.hpp +++ b/pgens/accretion/pgen.hpp @@ -5,13 +5,13 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "arch/traits.h" #include "utils/numeric.h" #include "archetypes/energy_dist.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "archetypes/spatial_dist.h" +#include "archetypes/traits.h" #include "framework/domain/metadomain.h" #include "kernels/particle_moments.hpp" @@ -43,7 +43,8 @@ namespace user { TWO * metric.spin() * g_00); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + Inline auto bx1(const coord_t& x_Ph) const + -> real_t { // at ( i , j + HALF ) coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -61,7 +62,8 @@ namespace user { } } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + Inline auto bx2(const coord_t& x_Ph) const + -> real_t { // at ( i + HALF , j ) coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -197,11 +199,17 @@ namespace user { template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator - static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; static constexpr auto metrics { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value }; - static constexpr auto dimensions { traits::compatible_with::value }; // for easy access to variables in the child class using arch::ProblemGenerator::D; diff --git a/pgens/examples/README.md b/pgens/examples/README.md new file mode 100644 index 000000000..93be01d40 --- /dev/null +++ b/pgens/examples/README.md @@ -0,0 +1,34 @@ +# Examples + +Problem generators in this directory are just examples demonstrating how to use some of the features of the Entity. + +- `custom_energy_distribution`: demonstrates how to initialize a uniform distribution of particles with two different custom-defined energy (velocity) distributions (per each species). + + image + +- `custom_spatial_distribution`: example of using the non-uniform plasma injector with a custom spatial distirbution + + https://github.com/user-attachments/assets/bea0c290-e7e4-4ec7-b360-68ce76beab5b + +- `match_fix_field_boundaries`: example of setting matching and/or fixed (coordinate-independent) field boundaries for the electromagnetic fields + + https://github.com/user-attachments/assets/a1b9ea22-34ce-474f-a9b0-49789a2e52b3 + +- `custom_emission`: simple example where two particles are initialized on gyrating trajectories probabilistically emitting photons while the total energy is conserved. + + https://github.com/user-attachments/assets/6c5c399a-bbab-4b93-be5f-35fed1959fcc + +- `external_fields`: example of using external fields for supplying a time-varying and species-dependent magnetic/electric fields and/or force-field imposed on particles. + + image + +- `atmosphere`: setting up a gravitationally bound "atmosphere" with a constant particle replenisher (the plot also highlights the importance of having a constant replenisher and a gravity force acting on the particles; both are enabled by default when using the `ATMOSPHERE` particle boundary conditions) + + atmosphere + +- `replenish_injector`: demonstration of how to use the replenish injector to periodically inject new plasma to a target density (both uniform and non-uniform) in `CustomPostStep` + + https://github.com/user-attachments/assets/4955f05e-4795-439f-a4a5-8196e42e987b + + https://github.com/user-attachments/assets/ebc303ed-7b21-4f97-9822-d81124fdf962 + diff --git a/pgens/examples/atmosphere/atmosphere.png b/pgens/examples/atmosphere/atmosphere.png new file mode 100644 index 000000000..0577f2c9e --- /dev/null +++ b/pgens/examples/atmosphere/atmosphere.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f922bdff068a435f024f314d3aad1ad25247dd16967015baf069a730017ceb81 +size 94964 diff --git a/pgens/examples/atmosphere/atmosphere.py b/pgens/examples/atmosphere/atmosphere.py new file mode 100644 index 000000000..d64bc0cff --- /dev/null +++ b/pgens/examples/atmosphere/atmosphere.py @@ -0,0 +1,44 @@ +import nt2 +import matplotlib.pyplot as plt +import numpy as np + +plt.rcParams["text.usetex"] = True +plt.rcParams["figure.dpi"] = 300 +plt.rcParams["figure.figsize"] = (6, 2) +plt.rcParams["font.family"] = "serif" + +data_normal = nt2.Data("atmosphere") +data_no_g = nt2.Data("atmosphere_no_g") +data_no_inject = nt2.Data("atmosphere_no_reinject") + +for d in [data_normal, data_no_g, data_no_inject]: + d.fields.coords["xh"] = ( + d.fields.coords["x"] / d.attrs["grid.boundaries.atmosphere.height"] + ) + +t = 1 + +data_normal.fields.N_1_2.sel(t=t, method="nearest").plot(label="normal", x="xh", lw=1) +data_no_g.fields.N_1_2.sel(t=t, method="nearest").plot(label="no gravity", x="xh", lw=1) +data_no_inject.fields.N_1_2.sel(t=t, method="nearest").plot( + label="no reinjection", x="xh", lw=1 +) + +xs = np.linspace(0, 20, 100) +plt.plot( + xs, + data_normal.attrs["grid.boundaries.atmosphere.density"] * np.exp(-xs), + label=r"$n_{\rm max} \exp{\{-x/h\}}$", + lw=1, + ls="--", + c="k", +) +plt.title(rf"$t={{{t:.2f}}}$") +plt.ylabel(r"$n_-+n_+$") +plt.xlabel(r"$x/h$") +plt.xlim(0, 20) +plt.yscale("log") +plt.legend() +plt.ylim(1e-2, 20) + +plt.savefig("atmosphere.png", bbox_inches="tight") diff --git a/pgens/examples/atmosphere/atmosphere.toml b/pgens/examples/atmosphere/atmosphere.toml new file mode 100644 index 000000000..a3d8d828d --- /dev/null +++ b/pgens/examples/atmosphere/atmosphere.toml @@ -0,0 +1,64 @@ +[simulation] + name = "atmosphere_no_g" + engine = "srpic" + runtime = 1.0 + +[grid] + resolution = [2048] + extent = [[0.0, 2.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["ATMOSPHERE", "FIXED"]] + particles = [["ATMOSPHERE", "REFLECT"]] + + [grid.boundaries.atmosphere] + temperature = 0.1 + density = 10.0 + height = 0.1 + species = [1, 2] + ds = 0.7 + +[scales] + larmor0 = 0.01 + skindepth0 = 0.01 + +[algorithms] + + [algorithms.deposit] + enable = false + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 128.0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e5 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e5 + +[output] + interval = 10 + + [output.fields] + quantities = ["N_1_2"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/atmosphere/pgen.hpp b/pgens/examples/atmosphere/pgen.hpp new file mode 100644 index 000000000..65cb7b68e --- /dev/null +++ b/pgens/examples/atmosphere/pgen.hpp @@ -0,0 +1,37 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "archetypes/problem_generator.h" +#include "archetypes/traits.h" +#include "framework/domain/metadomain.h" + +namespace user { + using namespace ntt; + + template + struct PGen : public arch::ProblemGenerator { + + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value + }; + + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + inline PGen(const SimulationParams& p, const Metadomain& metadomain) + : arch::ProblemGenerator { p } {} + }; + +} // namespace user + +#endif diff --git a/pgens/examples/custom_emission/custom_emission.mp4 b/pgens/examples/custom_emission/custom_emission.mp4 new file mode 100644 index 000000000..2efaca769 --- /dev/null +++ b/pgens/examples/custom_emission/custom_emission.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:320b85667465db5f2537a67f95462aeca96caa6cff2f80ae3fb57abfa09c95d4 +size 2896072 diff --git a/pgens/examples/custom_emission/custom_emission.py b/pgens/examples/custom_emission/custom_emission.py new file mode 100644 index 000000000..ae5d9b3be --- /dev/null +++ b/pgens/examples/custom_emission/custom_emission.py @@ -0,0 +1,51 @@ +import nt2 +import matplotlib.pyplot as plt +import numpy as np + +plt.rcParams["text.usetex"] = True + +data = nt2.Data("custom_emission") + + +def plot(t, data): + fig = plt.figure(figsize=(8, 4), dpi=150) + ax1 = fig.add_subplot(121) + prtls = data.particles.sel(t=slice(t - 0.5, t)).load() + prtls.plot.scatter( + ax=ax1, + x="x", + y="y", + color=["r" if sp == 1 else "b" for sp in prtls["sp"]], + ec=None, + alpha=(1 - (t - prtls["t"]) / 0.5) ** 2, + s=(1 - (t - prtls["t"]) / 0.5) * 5, + ) + ax1.set(xlabel=r"$x$", ylabel=r"$y$", xlim=(-1, 1), ylim=(-1, 1), aspect=1) + + ax2 = fig.add_subplot(122) + prtls1 = data.particles.sel(sp=1, t=slice(None, t)).load() + prtls2 = data.particles.sel(sp=2, t=slice(None, t)).load() + e1 = ( + prtls1.assign( + e=np.sqrt(1 + prtls1["ux"] ** 2 + prtls1["uy"] ** 2 + prtls1["uz"] ** 2) + ) + .groupby("t", as_index=False)["e"] + .sum() + ) + e2 = ( + prtls2.assign( + e=np.sqrt(prtls2["ux"] ** 2 + prtls2["uy"] ** 2 + prtls2["uz"] ** 2) + ) + .groupby("t", as_index=False)["e"] + .sum() + ) + e = e1.merge(e2, on="t", how="outer", suffixes=("_1", "_2")).fillna(0) + ax2.plot(e.t, e.e_1, label="emitters", c="r") + ax2.plot(e.t, e.e_2, label="emitted", c="b") + ax2.plot(e.t, e.e_1 + e.e_2, label="total", c="k") + ax2.set(xlabel=r"$t$", ylabel=r"total energy", xlim=(0, 5), ylim=(0, 3)) + ax2.axvline(t, color="gray", ls="--") + ax2.legend(loc="center left") + + +data.makeMovie(plot, framerate=30) diff --git a/pgens/examples/custom_emission/custom_emission.toml b/pgens/examples/custom_emission/custom_emission.toml new file mode 100644 index 000000000..fd4cd135e --- /dev/null +++ b/pgens/examples/custom_emission/custom_emission.toml @@ -0,0 +1,68 @@ +[simulation] + name = "custom_emission" + engine = "srpic" + runtime = 5.0 + +[grid] + resolution = [64, 64] + extent = [[-1.0, 1.0], [-1.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["PERIODIC"], ["PERIODIC"]] + +[scales] + larmor0 = 0.25 + skindepth0 = 1.0 + +[algorithms] + + [algorithms.deposit] + enable = false + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "p" + mass = 1.0 + charge = 1.0 + maxnpart = 1e2 + emission = "custom" + + [[particles.species]] + label = "ph" + mass = 0.0 + charge = 0.0 + maxnpart = 1e4 + +[setup] + x1_arr = [0.0, 0.0] + x2_arr = [0.0, 0.0] + x3_arr = [0.0, 0.0] + ux1_arr = [0.0, 0.0] + ux2_arr = [1.0, -1.0] + ux3_arr = [0.0, 0.0] + emission_probability = 0.05 + +[output] + interval = 1 + + [output.fields] + quantities = ["N_1", "N_2", "B"] + + [output.particles] + enable = true + stride = 1 + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/custom_emission/pgen.hpp b/pgens/examples/custom_emission/pgen.hpp new file mode 100644 index 000000000..e1d200bab --- /dev/null +++ b/pgens/examples/custom_emission/pgen.hpp @@ -0,0 +1,240 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" + +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "archetypes/traits.h" +#include "framework/domain/metadomain.h" +#include "kernels/emission/traits.h" +#include "kernels/injectors.hpp" + +#include + +#include +#include +#include + +namespace user { + using namespace ntt; + + template + struct InitFields { + Inline auto bx3(const coord_t&) const -> real_t { + return ONE; + } + }; + + template + struct RandomEmission { + struct Payload { + real_t photon_energy { ZERO }; + }; + + random_number_pool_t random_pool; + const real_t probability; + + array_t inj_idx { "inj_idx" }; + const npart_t inj_offset; + + array_t inj_i1, inj_i2, inj_i3; + array_t inj_dx1, inj_dx2, inj_dx3; + array_t inj_ux1, inj_ux2, inj_ux3; + array_t inj_phi; + array_t inj_weight; + array_t inj_tag; + array_t inj_pld_i; + + RandomEmission(random_number_pool_t& random_pool, + real_t probability, + npart_t inj_offset, + array_t& inj_i1, + array_t& inj_i2, + array_t& inj_i3, + array_t& inj_dx1, + array_t& inj_dx2, + array_t& inj_dx3, + array_t& inj_ux1, + array_t& inj_ux2, + array_t& inj_ux3, + array_t& inj_phi, + array_t& inj_weight, + array_t& inj_tag, + array_t& inj_pld_i) + : random_pool { random_pool } + , probability { probability } + , inj_offset { inj_offset } + , inj_i1 { inj_i1 } + , inj_i2 { inj_i2 } + , inj_i3 { inj_i3 } + , inj_dx1 { inj_dx1 } + , inj_dx2 { inj_dx2 } + , inj_dx3 { inj_dx3 } + , inj_ux1 { inj_ux1 } + , inj_ux2 { inj_ux2 } + , inj_ux3 { inj_ux3 } + , inj_phi { inj_phi } + , inj_weight { inj_weight } + , inj_tag { inj_tag } + , inj_pld_i { inj_pld_i } {} + + Inline auto shouldEmit(const coord_t&, + const coord_t&, + const vec_t& u_Ph, + const vec_t&, + const vec_t&, + vec_t& delta_u_Ph, + Payload& payload) const -> Kokkos::pair { + auto generator = random_pool.get_state(); + const auto rnd = Random(generator); + random_pool.free_state(generator); + if (rnd < probability) { + delta_u_Ph[0] = -0.1 * u_Ph[0]; + delta_u_Ph[1] = -0.1 * u_Ph[1]; + delta_u_Ph[2] = -0.1 * u_Ph[2]; + + const auto uSqr = NORM_SQR(u_Ph[0], u_Ph[1], u_Ph[2]); + const auto gammaSqr = ONE + uSqr; + const auto delta_uSqr = NORM_SQR(delta_u_Ph[0], + delta_u_Ph[1], + delta_u_Ph[2]); + const auto u_dot_delta_u = DOT(u_Ph[0], + u_Ph[1], + u_Ph[2], + delta_u_Ph[0], + delta_u_Ph[1], + delta_u_Ph[2]); + payload.photon_energy = math::sqrt(gammaSqr) * + (math::sqrt(ONE + delta_uSqr / gammaSqr + + TWO * u_dot_delta_u / gammaSqr) - + ONE); + return { true, true }; + } + return { false, false }; + } + + Inline auto emit(const tuple_t& xi_Cd, + const tuple_t& dxi_Cd, + const vec_t& direction, + real_t, + real_t, + const Payload& payload) const -> void { + const auto inj_index = Kokkos::atomic_fetch_add(&inj_idx(), 1); + kernel::InjectParticle( + inj_offset + inj_index, + inj_i1, + inj_i2, + inj_i3, + inj_dx1, + inj_dx2, + inj_dx3, + inj_ux1, + inj_ux2, + inj_ux3, + inj_phi, + inj_weight, + inj_tag, + inj_pld_i, + xi_Cd, + dxi_Cd, + { payload.photon_energy * direction[0], + payload.photon_energy * direction[1], + payload.photon_energy * direction[2] }); + } + + auto emitted_species_indices() const -> std::vector { + return { 2u }; + } + + auto numbers_injected() const -> std::vector { + auto inj_idx_h = Kokkos::create_mirror_view(inj_idx); + Kokkos::deep_copy(inj_idx_h, inj_idx); + return { inj_idx_h() }; + } + }; + + template + struct PGen : public arch::ProblemGenerator { + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value + }; + + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const Metadomain& metadomain; + + const real_t emission_probability; + + InitFields init_flds {}; + + inline PGen(const SimulationParams& p, const Metadomain& metadomain) + : arch::ProblemGenerator { p } + , metadomain { metadomain } + , emission_probability { params.template get( + "setup.emission_probability") } { + static_assert(kernel::traits::emission::IsValid, M>, "RandomEmission does not satisfy the requirements of an emission policy"); + } + + inline auto EmissionPolicy(simtime_t, + spidx_t, + Domain& domain) const -> RandomEmission { + return RandomEmission { + domain.random_pool(), emission_probability, + domain.species[1].npart(), domain.species[1].i1, + domain.species[1].i2, domain.species[1].i3, + domain.species[1].dx1, domain.species[1].dx2, + domain.species[1].dx3, domain.species[1].ux1, + domain.species[1].ux2, domain.species[1].ux3, + domain.species[1].phi, domain.species[1].weight, + domain.species[1].tag, domain.species[1].pld_i + }; + } + + inline void InitPrtls(Domain& domain) { + const auto empty = std::vector {}; + const auto x1_arr = params.template get>( + "setup.x1_arr", + empty); + const auto x2_arr = params.template get>( + "setup.x2_arr", + empty); + const auto x3_arr = params.template get>( + "setup.x3_arr", + empty); + const auto ux1_arr = params.template get>( + "setup.ux1_arr", + empty); + const auto ux2_arr = params.template get>( + "setup.ux2_arr", + empty); + const auto ux3_arr = params.template get>( + "setup.ux3_arr", + empty); + + std::map> data_arr { + { "x1", x1_arr }, + { "x2", x2_arr }, + { "x3", x3_arr }, + { "ux1", ux1_arr }, + { "ux2", ux2_arr }, + { "ux3", ux3_arr } + }; + arch::InjectGlobally(metadomain, domain, 1u, data_arr); + } + }; + +} // namespace user + +#endif diff --git a/pgens/examples/custom_energy_distribution/custom_energy_distribution.png b/pgens/examples/custom_energy_distribution/custom_energy_distribution.png new file mode 100644 index 000000000..8c7feac93 --- /dev/null +++ b/pgens/examples/custom_energy_distribution/custom_energy_distribution.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46eafd476c205b74d70c232f99e0f042f4e09f190223d1403d98e89e98eb7531 +size 34359 diff --git a/pgens/examples/custom_energy_distribution/custom_energy_distribution.py b/pgens/examples/custom_energy_distribution/custom_energy_distribution.py new file mode 100644 index 000000000..fdbb082cb --- /dev/null +++ b/pgens/examples/custom_energy_distribution/custom_energy_distribution.py @@ -0,0 +1,22 @@ +import nt2 +import matplotlib.pyplot as plt +import numpy as np + +plt.rcParams["text.usetex"] = True + +data = nt2.Data("custom_energy_distribution") + +fig = plt.figure(figsize=(10, 5), dpi=150) +ax1 = fig.add_subplot(1, 2, 1) +ax1.set_facecolor("k") +data.particles.sel(sp=1).isel(t=0).phase_plot(ax=ax1, cmap="inferno") +ax1.set(xlabel="$x$", ylabel="$u_x$", ylim=(-1.5, 1.5)) + +ax2 = fig.add_subplot(1, 2, 2) +ax2.set_facecolor("k") +data.particles.sel(sp=2).isel(t=0).phase_plot( + ax=ax2, y_quantity=lambda df: df["uy"], cmap="inferno" +) +ax2.set(xlabel="$x$", ylabel="$u_y$", ylim=(-1.5, 1.5)) + +plt.show() diff --git a/pgens/examples/custom_energy_distribution/custom_energy_distribution.toml b/pgens/examples/custom_energy_distribution/custom_energy_distribution.toml new file mode 100644 index 000000000..ed850ba26 --- /dev/null +++ b/pgens/examples/custom_energy_distribution/custom_energy_distribution.toml @@ -0,0 +1,63 @@ +[simulation] + name = "custom_energy_distribution" + engine = "srpic" + runtime = 0.1 + +[grid] + resolution = [64, 64] + extent = [[-1.0, 1.0], [-1.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["PERIODIC"], ["PERIODIC"]] + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + + [algorithms.deposit] + enable = false + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 16.0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e5 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e5 + +[setup] + + temperature = 1e-1 + drift_amplitude = 0.5 + +[output] + interval = 1 + + [output.fields] + enable = false + + [output.particles] + enable = true + stride = 1 + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/custom_energy_distribution/pgen.hpp b/pgens/examples/custom_energy_distribution/pgen.hpp new file mode 100644 index 000000000..e4fc1232b --- /dev/null +++ b/pgens/examples/custom_energy_distribution/pgen.hpp @@ -0,0 +1,105 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "archetypes/traits.h" +#include "framework/domain/metadomain.h" + +namespace user { + using namespace ntt; + + template + struct CustomDistribution_1 { + static constexpr auto D = Dim; + + CustomDistribution_1(random_number_pool_t& random_pool, + real_t temperature, + real_t drift_amplitude, + real_t box_size) + : random_pool { random_pool } + , temperature { temperature } + , drift_amplitude { drift_amplitude } + , kx { static_cast(constant::TWO_PI) / box_size } {} + + // the only requirement for the energy distribution is to have this operator() + // that takes in the particle position and velocity (by reference) and + // modifies (sets) the velocity according to the desired distribution + Inline void operator()(const coord_t& x_Ph, vec_t& v) const { + // sample a static 3D maxwellian + drift in x1 direction with sinusoidal spatial dependence + // @NOTE: for relativistic drift, use the built-in drifting Maxwellian + arch::JuttnerSinge(v, temperature, random_pool); + v[0] += drift_amplitude * math::sin(x_Ph[0] * kx); + } + + random_number_pool_t random_pool; + const real_t temperature, drift_amplitude, kx; + }; + + template + struct CustomDistribution_2 { + static constexpr auto D = Dim; + + CustomDistribution_2(real_t drift_amplitude) + : drift_amplitude { drift_amplitude } {} + + Inline void operator()(const coord_t& x_Ph, vec_t& v) const { + // zero temperature + counterstreaming drifts in x2 + v[1] = drift_amplitude * ((x_Ph[0] < ZERO) ? ONE : -ONE); + } + + const real_t drift_amplitude; + }; + + template + struct PGen : public arch::ProblemGenerator { + + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value + }; + + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const Metadomain& metadomain; + + inline PGen(const SimulationParams& p, const Metadomain& metadomain) + : arch::ProblemGenerator { p } + , metadomain { metadomain } {} + + inline void InitPrtls(Domain& domain) { + const auto temperature = params.template get("setup.temperature"); + const auto drift_amplitude = params.template get( + "setup.drift_amplitude"); + const auto box_size = metadomain.mesh().extent(in::x1).second - + metadomain.mesh().extent(in::x1).first; + const auto edist1 = CustomDistribution_1 { domain.random_pool(), + temperature, + drift_amplitude, + box_size }; + const auto edist2 = CustomDistribution_2 { drift_amplitude }; + + // distributions are then passed to the particle injector function + arch::InjectUniform( + params, + domain, + { 1u, 2u }, + { edist1, edist2 }, + ONE); + } + }; + +} // namespace user + +#endif diff --git a/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.mp4 b/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.mp4 new file mode 100644 index 000000000..90bfd5068 --- /dev/null +++ b/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:113ae5e58c6fdb999201e4a2b810e72d7f98363fa87a69804e7aad5245b2a677 +size 21175296 diff --git a/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.py b/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.py new file mode 100644 index 000000000..f554dbf8c --- /dev/null +++ b/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.py @@ -0,0 +1,15 @@ +import nt2 +import matplotlib.pyplot as plt + +data = nt2.Data("custom_spatial_distribution") + +def plot(t, data): + plt.rcParams["text.usetex"] = True + plt.rcParams["figure.dpi"] = 150 + data.fields.N_1_2.sel(t=t).plot(vmin=0, vmax=1.5) + plt.gca().set(xlabel=r"$x$", ylabel=r"$y$", title=rf"$t={{{t:.2f}}}$") + plt.gcf().axes[1].set_ylabel(r"$n_1+n_2$") + plt.gca().set_aspect(1) + plt.tight_layout() + +data.makeMovie(plot, framerate=30) diff --git a/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.toml b/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.toml new file mode 100644 index 000000000..710758ce1 --- /dev/null +++ b/pgens/examples/custom_spatial_distribution/custom_spatial_distribution.toml @@ -0,0 +1,57 @@ +[simulation] + name = "custom_spatial_distribution" + engine = "srpic" + runtime = 2.0 + +[grid] + resolution = [128, 128] + extent = [[-1.5, 1.5], [-1.0, 2.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["REFLECT", "REFLECT"], ["REFLECT", "REFLECT"]] + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + + [algorithms.deposit] + enable = false + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 16.0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e6 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e6 + +[output] + interval = 1 + + [output.fields] + quantities = ["N_1_2"] + + [output.particles] + stride = 10 + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/custom_spatial_distribution/pgen.hpp b/pgens/examples/custom_spatial_distribution/pgen.hpp new file mode 100644 index 000000000..0ba434e54 --- /dev/null +++ b/pgens/examples/custom_spatial_distribution/pgen.hpp @@ -0,0 +1,75 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "archetypes/traits.h" +#include "framework/domain/metadomain.h" + +namespace user { + using namespace ntt; + + template + struct CustomSpatialDistribution { + static constexpr auto D = Dim; + + CustomSpatialDistribution() {} + + // the only requirement for the spatial distribution is to have this operator() + // that takes in a position and returns the number density in that region (in units of n0) + Inline auto operator()(const coord_t& x_Ph) const -> real_t { + const auto ndens = ONE - + (SQR(x_Ph[1] - math::pow(math::abs(x_Ph[0]), THIRD)) + + SQR(x_Ph[0])); + if (ndens < ZERO) { + return ZERO; + } + return ndens; + } + }; + + template + struct PGen : public arch::ProblemGenerator { + + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value + }; + + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + inline PGen(const SimulationParams& p, const Metadomain& metadomain) + : arch::ProblemGenerator { p } {} + + inline void InitPrtls(Domain& domain) { + const auto sdist = CustomSpatialDistribution {}; + const auto edist = arch::Maxwellian { domain.mesh.metric, + domain.random_pool(), + static_cast(0.1) }; + + // distributions are then passed to the nonuniform particle injector + // function (same energy distribution is used for both species) + arch::InjectNonUniform( + params, + domain, + { 1u, 2u }, + { edist, edist }, + sdist, + ONE); + } + }; + +} // namespace user + +#endif diff --git a/pgens/examples/external_fields/external_fields.png b/pgens/examples/external_fields/external_fields.png new file mode 100644 index 000000000..aaa530831 --- /dev/null +++ b/pgens/examples/external_fields/external_fields.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e0e9473b9659cbe8ef51d529d73a50c58a1869960a0559e92cc5a70bdd052ad +size 39577 diff --git a/pgens/examples/external_fields/external_fields.py b/pgens/examples/external_fields/external_fields.py new file mode 100644 index 000000000..8122d8aa6 --- /dev/null +++ b/pgens/examples/external_fields/external_fields.py @@ -0,0 +1,23 @@ +import nt2 +import matplotlib.pyplot as plt +import numpy as np + +plt.rcParams["text.usetex"] = True + +data = nt2.Data("external_fields") +prtls = data.particles.load() + +fig = plt.figure(figsize=(10, 5), dpi=150) +ax = fig.add_subplot(111) + +ax.scatter( + prtls["x"], + prtls["y"], + s=1, + color=np.choose(prtls["sp"].array - 1, ["r", "g", "b"]), + ec=None, + alpha=(100 - prtls["t"].array) / 100, +) +ax.set(xlim=(-1, 1), ylim=(-1, 1), xlabel=r"$x$", ylabel=r"$y$", aspect=1) + +plt.show() diff --git a/pgens/examples/external_fields/external_fields.toml b/pgens/examples/external_fields/external_fields.toml new file mode 100644 index 000000000..c6881f763 --- /dev/null +++ b/pgens/examples/external_fields/external_fields.toml @@ -0,0 +1,90 @@ +[simulation] + name = "external_fields" + engine = "srpic" + runtime = 5.0 + +[grid] + resolution = [64, 64] + extent = [[-1.0, 1.0], [-1.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["PERIODIC"], ["PERIODIC"]] + +[scales] + larmor0 = 0.25 + skindepth0 = 1.0 + +[algorithms] + + [algorithms.deposit] + enable = false + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 1.0 + + [[particles.species]] + label = "p1" + mass = 1.0 + charge = 1.0 + maxnpart = 1e2 + + [[particles.species]] + label = "p2" + mass = 1.0 + charge = 1.0 + maxnpart = 1e2 + + [[particles.species]] + label = "p3" + mass = 1.0 + charge = 1.0 + maxnpart = 1e2 + +[setup] + prtls = [ + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ], + [ + -0.5, + 0.0, + 0.0, + 0.0, + 0.0, + ], + [ + 0.5, + 0.0, + 0.0, + 0.0, + 0.0, + ], + ] + prtl_species = [1, 2, 3] + +[output] + interval = 1 + + [output.fields] + enable = false + + [output.particles] + enable = true + stride = 1 + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/external_fields/pgen.hpp b/pgens/examples/external_fields/pgen.hpp new file mode 100644 index 000000000..9f7eca314 --- /dev/null +++ b/pgens/examples/external_fields/pgen.hpp @@ -0,0 +1,145 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "archetypes/traits.h" +#include "framework/domain/metadomain.h" + +#include + +#include +#include +#include + +namespace user { + using namespace ntt; + + template + struct ExtFields { + ExtFields(simtime_t time, spidx_t sp) : time { time }, sp { sp } {} + + /* + * Particle's equation of motion is: + * + * du / dt = f_ext + (q / mc) * ((E + E_ext) + v x (B + B_ext)) + * + * in dimensionless terms: + * + * du / dt = f_ext + (q / m) / (q0 / m0) * omegaB0 * ((e + e_ext) + v x (b + b_ext)) + * + * - f_ext is the external force-field (acceleration) defined here + * - E and B are interpolated fields from the grid + * - e = E / B0 and b = B / B0 + * - e_ext = E_ext / B0 and b_ext = B_ext / B0 are the dimensionless external fields defined here + * + */ + + // f_ext: external force-field (acceleration): + Inline auto fx1(const coord_t&) const -> real_t { + return (sp % 2u == 0u) ? -HALF : HALF; + } + + // b_ext: external magnetic field: + Inline auto bx3(const coord_t&) const -> real_t { + return ONE + 0.2 * time; + } + + const simtime_t time; + const spidx_t sp; + }; + + template + struct PGen : public arch::ProblemGenerator { + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value + }; + + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const Metadomain& metadomain; + + inline PGen(const SimulationParams& p, const Metadomain& metadomain) + : arch::ProblemGenerator { p } + , metadomain { metadomain } {} + + /* + * @returns a pair of (apply_external_fields, external_fields) + * + * @note apply_external_fields is true for species other than 1 (i.e., 2 and 3 in this case) + */ + inline auto ExternalFields(simtime_t time, + spidx_t sp, + const Domain& domain) const + -> std::pair> { + // apply only to species 2 and 3 + return { + sp != 1u, + ExtFields { time, sp } + }; + } + + inline void InitPrtls(Domain& domain) { + const auto prtls = params.template get>>( + "setup.prtls"); + const auto prtl_species = params.template get>( + "setup.prtl_species"); + raise::ErrorIf(prtl_species.size() != prtls.size(), + "setup.prtls_species should be a vector of the same size " + "as setup.prtls", + HERE); + if (prtls.size() > 0u) { + raise::ErrorIf(prtls[0].size() != 3u + static_cast(D), + "setup.prtls should be a vector of vectors of size 3+D", + HERE); + for (auto p = 0u; p < prtls.size(); ++p) { + const auto prtl = prtls[p]; + const auto prtl_spec = prtl_species[p]; + std::map> data_arr; + data_arr["x1"] = { prtl[0] }; + if constexpr (D == Dim::_2D or D == Dim::_3D) { + data_arr["x2"] = { prtl[1] }; + } + if constexpr (D == Dim::_3D) { + data_arr["x3"] = { prtl[2] }; + } + if constexpr (D == Dim::_1D) { + data_arr["ux1"] = { prtl[1] }; + data_arr["ux2"] = { prtl[2] }; + data_arr["ux3"] = { prtl[3] }; + } + if constexpr (D == Dim::_2D) { + data_arr["ux1"] = { prtl[2] }; + data_arr["ux2"] = { prtl[3] }; + data_arr["ux3"] = { prtl[4] }; + } + if constexpr (D == Dim::_3D) { + data_arr["ux1"] = { prtl[3] }; + data_arr["ux2"] = { prtl[4] }; + data_arr["ux3"] = { prtl[5] }; + } + raise::ErrorIf( + prtl_spec <= 0 or prtl_spec > domain.species.size(), + "setup.prtl_species should be a vector of integers between 1 and " + "the number of species in the simulation", + HERE); + arch::InjectGlobally(metadomain, domain, prtl_spec, data_arr); + } + } + } + }; + +} // namespace user + +#endif diff --git a/pgens/examples/match_fix_field_boundaries/fix_field_boundaries.toml b/pgens/examples/match_fix_field_boundaries/fix_field_boundaries.toml new file mode 100644 index 000000000..ebcb9579c --- /dev/null +++ b/pgens/examples/match_fix_field_boundaries/fix_field_boundaries.toml @@ -0,0 +1,49 @@ +[simulation] + name = "fix_field_boundaries" + engine = "srpic" + runtime = 2.0 + +[grid] + resolution = [2048] + extent = [[0.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["FIXED", "FIXED"]] + particles = [["REFLECT", "ABSORB"]] + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 1.0 + +[setup] + + amplitude = 1.0 + omega = 500.0 + t_transition = 0.1 + t_duration = 0.25 + +[output] + interval_time = 0.01 + + [output.fields] + quantities = ["E", "B"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/match_fix_field_boundaries/match_field_boundaries.toml b/pgens/examples/match_fix_field_boundaries/match_field_boundaries.toml new file mode 100644 index 000000000..97c476042 --- /dev/null +++ b/pgens/examples/match_fix_field_boundaries/match_field_boundaries.toml @@ -0,0 +1,52 @@ +[simulation] + name = "match_field_boundaries" + engine = "srpic" + runtime = 2.0 + +[grid] + resolution = [2048] + extent = [[0.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["FIXED", "MATCH"]] + particles = [["REFLECT", "ABSORB"]] + + [grid.boundaries.match] + ds = 0.05 + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 1.0 + +[setup] + + amplitude = 1.0 + omega = 500.0 + t_transition = 0.1 + t_duration = 0.25 + +[output] + interval_time = 0.01 + + [output.fields] + quantities = ["E", "B"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/match_fix_field_boundaries/match_fix_field_boundaries.mp4 b/pgens/examples/match_fix_field_boundaries/match_fix_field_boundaries.mp4 new file mode 100644 index 000000000..eeac195bf --- /dev/null +++ b/pgens/examples/match_fix_field_boundaries/match_fix_field_boundaries.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beefe42c1821c23ab59a90f4d5db7d125ef3fc4ddf504ae4f6d4dd97f9e77996 +size 17354737 diff --git a/pgens/examples/match_fix_field_boundaries/match_fix_field_boundaries.py b/pgens/examples/match_fix_field_boundaries/match_fix_field_boundaries.py new file mode 100644 index 000000000..ae591a093 --- /dev/null +++ b/pgens/examples/match_fix_field_boundaries/match_fix_field_boundaries.py @@ -0,0 +1,76 @@ +import nt2 +import matplotlib.pyplot as plt + +data_fix = nt2.Data("fix_field_boundaries") +data_match = nt2.Data("match_field_boundaries") + + +def plot(t, _): + plt.rcParams["text.usetex"] = True + omega = data_fix.attrs["setup.omega"] + + data_fix.fields.coords["xo"] = data_fix.fields.coords["x"] * omega + data_match.fields.coords["xo"] = data_match.fields.coords["x"] * omega + xmax = data_match.fields.coords["xo"].max().values[()] + ds = data_match.attrs["grid.boundaries.match.ds"][0] * omega + + fig = plt.figure(figsize=(10, 5), dpi=200) + ax1 = fig.add_subplot(211) + data_fix.fields.Ez.sel(t=t, method="nearest").plot(ax=ax1, label="$E_z$", x="xo") + data_fix.fields.By.sel(t=t, method="nearest").plot(ax=ax1, label="$B_y$", x="xo") + + ax2 = fig.add_subplot(212) + data_match.fields.Ez.sel(t=t, method="nearest").plot(ax=ax2, label="$E_z$", x="xo") + data_match.fields.By.sel(t=t, method="nearest").plot(ax=ax2, label="$B_y$", x="xo") + + for ax in [ax1, ax2]: + ax.set( + xlim=(0, omega), + ylim=(-1.1, 1.1), + ylabel="", + title=None, + ) + + ax1.set(xticklabels=[], xlabel="") + ax2.set(xlabel=r"$x$ [$c / \omega$]") + ax1.set(title=rf"$\omega t={omega * t:.1f}$") + + for ax, label in [(ax1, "FIXED"), (ax2, "MATCH")]: + ax.text( + 0.99, + 0.94, + r"\texttt{" + label + "}", + transform=ax.transAxes, + ha="right", + va="top", + fontsize=8, + ) + + plt.subplots_adjust(hspace=0.35) + + mid_y = (ax1.get_position().y0 + ax2.get_position().y1) / 2 + right_x = ax1.get_position().x1 + + handles, labels = ax1.get_legend_handles_labels() + leg = fig.legend( + handles, + labels, + loc="center right", + bbox_to_anchor=(right_x, mid_y), + bbox_transform=fig.transFigure, + ncol=1, + frameon=True, + ) + leg.set_zorder(10) + ax2.fill_between( + [xmax - ds, xmax], + -1.1, + 1.1, + color="grey", + alpha=0.3, + transform=ax2.get_xaxis_transform(), + ) + plt.tight_layout() + + +data_fix.makeMovie(plot, framerate=15) diff --git a/pgens/examples/match_fix_field_boundaries/pgen.hpp b/pgens/examples/match_fix_field_boundaries/pgen.hpp new file mode 100644 index 000000000..2adcaa944 --- /dev/null +++ b/pgens/examples/match_fix_field_boundaries/pgen.hpp @@ -0,0 +1,110 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "archetypes/problem_generator.h" +#include "archetypes/traits.h" +#include "framework/domain/metadomain.h" + +namespace user { + using namespace ntt; + + template + struct ZeroFields { + auto ex2(const coord_t&) const -> real_t { + return ZERO; + } + + auto ex3(const coord_t&) const -> real_t { + return ZERO; + } + + auto bx1(const coord_t&) const -> real_t { + return ZERO; + } + }; + + template + struct PGen : public arch::ProblemGenerator { + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value + }; + + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + const real_t amplitude, omega; + const real_t t_transition, t_duration; + + inline PGen(const SimulationParams& p, const Metadomain&) + : arch::ProblemGenerator { p } + , amplitude { p.template get("setup.amplitude", ONE) } + , omega { p.template get("setup.omega", ONE) } + , t_transition { p.template get("setup.t_transition") } + , t_duration { p.template get("setup.t_duration") } {} + + /** + * Sets up the driving field on the left boundary. + * + * @param bc_in Direction of the boundary (only be used for one side, so no need to check) + * @param comp Electromagnetic component to set + * + * @note Because the wave is only set on the boundary, no coordinate dependency is needed. + * + * @note The fields are normalized to B0 (nominal magnetic field) + * @note Launching an Ez x By wave from the left boundary + * @note else-statement will only be hit if right boundary is also FIXED + * + * @return Pair of (value to set, whether to set it or not) + */ + auto FixFieldsConst(simtime_t time, const bc_in& bc, em comp) const + -> std::pair { + if (bc == bc_in::Mx1) { + const auto phase { time * omega }; + real_t ampl { ZERO }; + if (time < t_transition) { + ampl = (time / t_transition); + } else if (time < t_transition + t_duration) { + ampl = ONE; + } else { + ampl = math::max( + ONE - (static_cast(time) - t_transition - t_duration) / + t_transition, + ZERO); + } + ampl *= amplitude; + + if (comp == em::ex3) { + return { -ampl * math::cos(phase), true }; + } else if (comp == em::bx2) { + return { math::sin(phase) * ampl, true }; + } else { + return { ZERO, true }; + } + } else { + return { ZERO, true }; + } + } + + /* + * Enough to only enforce Ey, Ez and Bx to zero on the right boundary + * + * @note Only called if one of the boundaries is MATCH + */ + auto MatchFields(simtime_t) const -> ZeroFields { + return ZeroFields {}; + } + }; + +} // namespace user + +#endif diff --git a/pgens/examples/replenish_injector/nonuniform_replenish.mp4 b/pgens/examples/replenish_injector/nonuniform_replenish.mp4 new file mode 100644 index 000000000..139451d66 --- /dev/null +++ b/pgens/examples/replenish_injector/nonuniform_replenish.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec180a8026198809970dd325a07e332a431019ceae357495c702e200674b4d6a +size 15809969 diff --git a/pgens/examples/replenish_injector/nonuniform_replenish.toml b/pgens/examples/replenish_injector/nonuniform_replenish.toml new file mode 100644 index 000000000..77afab39b --- /dev/null +++ b/pgens/examples/replenish_injector/nonuniform_replenish.toml @@ -0,0 +1,57 @@ +[simulation] + name = "nonuniform_replenish" + engine = "srpic" + runtime = 2.0 + +[grid] + resolution = [128, 128] + extent = [[0.0, 1.0], [0.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["ABSORB", "ABSORB"], ["PERIODIC"]] + +[scales] + larmor0 = 0.01 + skindepth0 = 0.1 + +[algorithms] + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 8.0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e6 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e6 + +[setup] + target_density = "nonuniform" + +[output] + interval = 10 + + [output.fields] + quantities = ["N_1_2"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/examples/replenish_injector/pgen.hpp b/pgens/examples/replenish_injector/pgen.hpp new file mode 100644 index 000000000..8c196140f --- /dev/null +++ b/pgens/examples/replenish_injector/pgen.hpp @@ -0,0 +1,155 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/problem_generator.h" +#include "archetypes/spatial_dist.h" +#include "archetypes/traits.h" +#include "framework/domain/metadomain.h" +#include "kernels/particle_moments.hpp" + +namespace user { + using namespace ntt; + + template + struct EMFields { + Inline auto ex2(const coord_t&) const -> real_t { + return 0.8; + } + + Inline auto bx3(const coord_t&) const -> real_t { + return ONE; + } + }; + + template + struct NonUniformTargetDensity { + Inline auto operator()(const coord_t& x_Ph) const -> real_t { + // example of a non-uniform target density that peaks at the center of the domain + real_t r2 { ZERO }; + for (auto d = 0u; d < D; ++d) { + r2 += SQR(x_Ph[d] - 0.5); + } + return std::exp( + -r2 / SQR(0.2)); // <-- characteristic width of the density profile + } + }; + + template + struct PGen : public arch::ProblemGenerator { + + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value + }; + + using arch::ProblemGenerator::D; + using arch::ProblemGenerator::C; + using arch::ProblemGenerator::params; + + EMFields init_flds; + + const std::string target_density; + + inline PGen(const SimulationParams& p, const Metadomain& metadomain) + : arch::ProblemGenerator { p } + , target_density { params.template get( + "setup.target_density") } {} + + void CustomPostStep(timestep_t step, simtime_t time, Domain& domain) { + if (step % 100u != 0u) { + return; + } + // perform replenishment and injection every 100 timesteps + + { // compute density of species #1 and #2 + + // saves the density to domain.fields.buff(:,:,:,0) + const auto ni2 = domain.mesh.n_active(in::x2); + const auto inv_n0 = ONE / params.template get("scales.n0"); + + auto scatter_buff = Kokkos::Experimental::create_scatter_view( + domain.fields.buff); + Kokkos::deep_copy(domain.fields.buff, ZERO); + for (const auto sp : std::vector { 1, 2 }) { + const auto& prtl_spec = domain.species[sp - 1]; + Kokkos::parallel_for("ComputeDensity", + prtl_spec.rangeActiveParticles(), + kernel::ParticleMoments_kernel( + {}, + scatter_buff, + 0u, + prtl_spec.i1, + prtl_spec.i2, + prtl_spec.i3, + prtl_spec.dx1, + prtl_spec.dx2, + prtl_spec.dx3, + prtl_spec.ux1, + prtl_spec.ux2, + prtl_spec.ux3, + prtl_spec.phi, + prtl_spec.weight, + prtl_spec.tag, + prtl_spec.mass(), + prtl_spec.charge(), + false, + domain.mesh.metric, + domain.mesh.flds_bc(), + ni2, + inv_n0, + 0u)); + } + Kokkos::Experimental::contribute(domain.fields.buff, scatter_buff); + } + + const auto energy_dist = arch::Maxwellian( + domain.mesh.metric, + domain.random_pool(), + 0.2); // <-- target temperature for injection + if (target_density == "uniform") { + // pass the computed density to the replenisher + const auto replenish_sdist = arch::ReplenishUniform( + domain.mesh.metric, + domain.fields.buff, + 0u, // <-- index in buff where the density is stored + ONE); // <-- target density for replenishment + arch::InjectNonUniform( + params, + domain, + { 1, 2 }, + { energy_dist, energy_dist }, + replenish_sdist, + ONE); + } else { + const auto target_density_profile = NonUniformTargetDensity {}; + const auto replenish_sdist = + arch::Replenish( + domain.mesh.metric, + domain.fields.buff, + 0u, // <-- index in buff where the density is stored + target_density_profile, + ONE); // <-- target density for replenishment + arch::InjectNonUniform( + params, + domain, + { 1, 2 }, + { energy_dist, energy_dist }, + replenish_sdist, + ONE); + } + } + }; + +} // namespace user + +#endif diff --git a/pgens/examples/replenish_injector/replenish_injector.py b/pgens/examples/replenish_injector/replenish_injector.py new file mode 100644 index 000000000..dad9f8f16 --- /dev/null +++ b/pgens/examples/replenish_injector/replenish_injector.py @@ -0,0 +1,66 @@ +import nt2 +import matplotlib.pyplot as plt + + +def plot(t, data): + plt.rcParams["text.usetex"] = True + plt.rcParams["font.family"] = "serif" + plt.rcParams["figure.dpi"] = 250 + ax = plt.gca() + data.fields.N_1_2.sel(t=t, method="nearest").plot( + vmin=0, vmax=1.5, ax=ax, cmap="inferno" + ) + plt.gcf().axes[1].set_ylabel(r"$n_-+n_+$") + + ax.annotate( + "", + xy=(0.08, 0.75), + xycoords="axes fraction", + xytext=(0.08, 0.55), + textcoords="axes fraction", + arrowprops=dict(arrowstyle="-|>", color="#e05c2e", lw=1.8), + ) + ax.text( + 0.095, + 0.65, + r"$E_y$", + transform=ax.transAxes, + color="#e05c2e", + fontsize=12, + va="center", + ) + + circle = plt.Circle( + (0.08, 0.35), + 0.035, + transform=ax.transAxes, + fill=False, + color="#2e80e0", + lw=1.8, + clip_on=False, + ) + ax.add_patch(circle) + ax.plot(0.08, 0.35, ".", transform=ax.transAxes, color="#2e80e0", markersize=6) + ax.text( + 0.125, + 0.35, + r"$B_z$", + transform=ax.transAxes, + color="#2e80e0", + fontsize=12, + va="center", + ) + ax.set( + xlabel=r"$x$", + ylabel=r"$y$", + title=rf"$t={{{t:.2f}}}$", + xlim=(0, 1), + ylim=(0, 1), + ) + + +data_uniform_replenish = nt2.Data("uniform_replenish") +data_uniform_replenish.makeMovie(plot, framerate=10) + +data_nonuniform_replenish = nt2.Data("nonuniform_replenish") +data_nonuniform_replenish.makeMovie(plot, framerate=10) diff --git a/pgens/examples/replenish_injector/uniform_replenish.mp4 b/pgens/examples/replenish_injector/uniform_replenish.mp4 new file mode 100644 index 000000000..ae5669087 --- /dev/null +++ b/pgens/examples/replenish_injector/uniform_replenish.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72668cf43d686e4ee9c9085243c41dbcc9e5845f1c097d70ca7a8263d26548b3 +size 33877523 diff --git a/pgens/examples/replenish_injector/uniform_replenish.toml b/pgens/examples/replenish_injector/uniform_replenish.toml new file mode 100644 index 000000000..37977d08e --- /dev/null +++ b/pgens/examples/replenish_injector/uniform_replenish.toml @@ -0,0 +1,57 @@ +[simulation] + name = "uniform_replenish" + engine = "srpic" + runtime = 2.0 + +[grid] + resolution = [128, 128] + extent = [[0.0, 1.0], [0.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["ABSORB", "ABSORB"], ["PERIODIC"]] + +[scales] + larmor0 = 0.01 + skindepth0 = 0.1 + +[algorithms] + + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 8.0 + + [[particles.species]] + label = "e+" + mass = 1.0 + charge = 1.0 + maxnpart = 1e6 + + [[particles.species]] + label = "e-" + mass = 1.0 + charge = -1.0 + maxnpart = 1e6 + +[setup] + target_density = "uniform" + +[output] + interval = 10 + + [output.fields] + quantities = ["N_1_2"] + + [output.particles] + enable = false + + [output.spectra] + enable = false + +[checkpoint] + keep = 0 diff --git a/pgens/magnetosphere/pgen.hpp b/pgens/magnetosphere/pgen.hpp index 1afc8ccc4..706c35018 100644 --- a/pgens/magnetosphere/pgen.hpp +++ b/pgens/magnetosphere/pgen.hpp @@ -5,10 +5,10 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "arch/traits.h" #include "utils/numeric.h" #include "archetypes/problem_generator.h" +#include "archetypes/traits.h" #include "framework/domain/metadomain.h" #include @@ -52,13 +52,13 @@ namespace user { template struct DriveFields : public InitFields { - DriveFields(real_t time, + DriveFields(simtime_t time, real_t bsurf, real_t rstar, real_t omega, const std::string& field_geometry) : InitFields { bsurf, rstar, field_geometry } - , time { time } + , time { (real_t)time } , Omega { omega } {} using InitFields::bx1; @@ -87,11 +87,15 @@ namespace user { template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator - static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; static constexpr auto metrics { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value }; - static constexpr auto dimensions { traits::compatible_with::value }; // for easy access to variables in the child class using arch::ProblemGenerator::D; @@ -113,11 +117,11 @@ namespace user { inline PGen() {} - auto AtmFields(real_t time) const -> DriveFields { + auto AtmFields(simtime_t time) const -> DriveFields { return DriveFields { time, Bsurf, Rstar, Omega, field_geom }; } - auto MatchFields(real_t) const -> InitFields { + auto MatchFields(simtime_t) const -> InitFields { return InitFields { Bsurf, Rstar, field_geom }; } }; diff --git a/pgens/magnetosphere/sketch.png b/pgens/magnetosphere/sketch.png index fe33228e4..18ddc7199 100644 Binary files a/pgens/magnetosphere/sketch.png and b/pgens/magnetosphere/sketch.png differ diff --git a/pgens/pgen.hpp b/pgens/pgen.hpp index 78d8e6486..d2d091ebf 100644 --- a/pgens/pgen.hpp +++ b/pgens/pgen.hpp @@ -4,10 +4,10 @@ #include "enums.h" #include "global.h" -#include "arch/traits.h" #include "utils/formatting.h" #include "archetypes/problem_generator.h" +#include "archetypes/traits.h" #include "framework/domain/metadomain.h" #include @@ -19,18 +19,18 @@ namespace user { struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator static constexpr auto engines { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value }; static constexpr auto metrics { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value }; static constexpr auto dimensions { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value }; // for easy access to variables in the child class diff --git a/pgens/reconnection/pgen.hpp b/pgens/reconnection/pgen.hpp index d152b578a..4711aec22 100644 --- a/pgens/reconnection/pgen.hpp +++ b/pgens/reconnection/pgen.hpp @@ -5,16 +5,15 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "arch/traits.h" #include "utils/numeric.h" #include "archetypes/energy_dist.h" #include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" #include "archetypes/spatial_dist.h" +#include "archetypes/traits.h" #include "archetypes/utils.h" #include "framework/domain/metadomain.h" - #include "kernels/particle_moments.hpp" namespace user { @@ -141,10 +140,14 @@ namespace user { template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator - static constexpr auto engines { traits::compatible_with::value }; - static constexpr auto metrics { traits::compatible_with::value }; + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; static constexpr auto dimensions { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value }; // for easy access to variables in the child class diff --git a/pgens/reconnection/sketch.png b/pgens/reconnection/sketch.png index 6b25615d3..1d0c76b65 100644 Binary files a/pgens/reconnection/sketch.png and b/pgens/reconnection/sketch.png differ diff --git a/pgens/shock/pgen.hpp b/pgens/shock/pgen.hpp index fc579777d..b8289c9a3 100644 --- a/pgens/shock/pgen.hpp +++ b/pgens/shock/pgen.hpp @@ -4,12 +4,12 @@ #include "enums.h" #include "global.h" -#include "arch/traits.h" #include "utils/error.h" #include "utils/numeric.h" #include "archetypes/field_setter.h" #include "archetypes/problem_generator.h" +#include "archetypes/traits.h" #include "archetypes/utils.h" #include "framework/domain/metadomain.h" @@ -69,10 +69,14 @@ namespace user { template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator - static constexpr auto engines { traits::compatible_with::value }; - static constexpr auto metrics { traits::compatible_with::value }; + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; + static constexpr auto metrics { + arch::traits::pgen::compatible_with::value + }; static constexpr auto dimensions { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value }; // for easy access to variables in the child class @@ -113,12 +117,12 @@ namespace user { inline PGen() {} - auto MatchFields(real_t time) const -> InitFields { + auto MatchFields(simtime_t) const -> InitFields { return init_flds; } - auto FixFieldsConst(const bc_in&, - const em& comp) const -> std::pair { + auto FixFieldsConst(const bc_in&, const em& comp) const + -> std::pair { if (comp == em::ex1) { return { init_flds.ex1({ ZERO }), true }; } else if (comp == em::ex2) { diff --git a/pgens/shock/sketch.png b/pgens/shock/sketch.png index 9df562721..64ebce780 100644 Binary files a/pgens/shock/sketch.png and b/pgens/shock/sketch.png differ diff --git a/pgens/streaming/pgen.hpp b/pgens/streaming/pgen.hpp index dca6cc31d..9c87d7d97 100644 --- a/pgens/streaming/pgen.hpp +++ b/pgens/streaming/pgen.hpp @@ -5,11 +5,11 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "arch/traits.h" #include "utils/error.h" #include "utils/numeric.h" #include "archetypes/problem_generator.h" +#include "archetypes/traits.h" #include "archetypes/utils.h" #include "framework/domain/domain.h" #include "framework/domain/metadomain.h" @@ -54,18 +54,20 @@ namespace user { struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator - static constexpr auto engines = traits::compatible_with::value; - static constexpr auto metrics = traits::compatible_with::value; + static constexpr auto engines = + arch::traits::pgen::compatible_with::value; + static constexpr auto metrics = + arch::traits::pgen::compatible_with::value; static constexpr auto dimensions = - traits::compatible_with::value; + arch::traits::pgen::compatible_with::value; // for easy access to variables in the child class using arch::ProblemGenerator::D; using arch::ProblemGenerator::C; using arch::ProblemGenerator::params; - prmvec_t drifts_in_x, drifts_in_y, drifts_in_z; - prmvec_t densities, temperatures; + prmvec_t drifts_in_x, drifts_in_y, drifts_in_z; + prmvec_t densities, temperatures; // initial magnetic field real_t Btheta, Bphi, Bmag; InitFields init_flds; diff --git a/pgens/streaming/sketch.png b/pgens/streaming/sketch.png index a2cc66102..82ea02092 100644 Binary files a/pgens/streaming/sketch.png and b/pgens/streaming/sketch.png differ diff --git a/pgens/turbulence/pgen.hpp b/pgens/turbulence/pgen.hpp index dce59f029..17093d86a 100644 --- a/pgens/turbulence/pgen.hpp +++ b/pgens/turbulence/pgen.hpp @@ -9,8 +9,8 @@ #include "utils/numeric.h" #include "archetypes/energy_dist.h" -#include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" +#include "archetypes/traits.h" #include "archetypes/utils.h" #include "framework/domain/domain.h" #include "framework/domain/metadomain.h" @@ -293,9 +293,12 @@ namespace user { struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator - static constexpr auto engines = traits::compatible_with::value; - static constexpr auto metrics = traits::compatible_with::value; - static constexpr auto dimensions = traits::compatible_with::value; + static constexpr auto engines = + arch::traits::pgen::compatible_with::value; + static constexpr auto metrics = + arch::traits::pgen::compatible_with::value; + static constexpr auto dimensions = + arch::traits::pgen::compatible_with::value; // for easy access to variables in the child class using arch::ProblemGenerator::D; diff --git a/pgens/wald/pgen.hpp b/pgens/wald/pgen.hpp index 71ee905e3..292f6f7fe 100644 --- a/pgens/wald/pgen.hpp +++ b/pgens/wald/pgen.hpp @@ -5,21 +5,16 @@ #include "global.h" #include "arch/kokkos_aliases.h" -#include "arch/traits.h" #include "utils/comparators.h" #include "utils/error.h" #include "utils/formatting.h" -#include "utils/log.h" #include "utils/numeric.h" -#include "archetypes/energy_dist.h" -#include "archetypes/particle_injector.h" #include "archetypes/problem_generator.h" -#include "framework/domain/domain.h" +#include "archetypes/traits.h" #include "framework/domain/metadomain.h" #include -#include enum InitFieldGeometry { Wald, @@ -64,7 +59,8 @@ namespace user { TWO * metric.spin() * g_00); } - Inline auto bx1(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + Inline auto bx1(const coord_t& x_Ph) const + -> real_t { // at ( i , j + HALF ) coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -82,7 +78,8 @@ namespace user { } } - Inline auto bx2(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + Inline auto bx2(const coord_t& x_Ph) const + -> real_t { // at ( i + HALF , j ) coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -99,8 +96,8 @@ namespace user { } } - Inline auto bx3( - const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j + HALF ) + Inline auto bx3(const coord_t& x_Ph) const + -> real_t { // at ( i + HALF , j + HALF ) if (field_geometry == InitFieldGeometry::Wald) { coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -120,7 +117,8 @@ namespace user { } } - Inline auto dx1(const coord_t& x_Ph) const -> real_t { // at ( i + HALF , j ) + Inline auto dx1(const coord_t& x_Ph) const + -> real_t { // at ( i + HALF , j ) if (field_geometry == InitFieldGeometry::Wald) { coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -158,7 +156,8 @@ namespace user { } } - Inline auto dx2(const coord_t& x_Ph) const -> real_t { // at ( i , j + HALF ) + Inline auto dx2(const coord_t& x_Ph) const + -> real_t { // at ( i , j + HALF ) if (field_geometry == InitFieldGeometry::Wald) { coord_t xi { ZERO }, x0m { ZERO }, x0p { ZERO }; metric.template convert(x_Ph, xi); @@ -232,11 +231,17 @@ namespace user { template struct PGen : public arch::ProblemGenerator { // compatibility traits for the problem generator - static constexpr auto engines { traits::compatible_with::value }; + static constexpr auto engines { + arch::traits::pgen::compatible_with::value + }; static constexpr auto metrics { - traits::compatible_with::value + arch::traits::pgen::compatible_with::value + }; + static constexpr auto dimensions { + arch::traits::pgen::compatible_with::value }; - static constexpr auto dimensions { traits::compatible_with::value }; // for easy access to variables in the child class using arch::ProblemGenerator::D; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 31114c330..443dc0545 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,15 +24,6 @@ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -# dependencies -add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global) -add_subdirectory(${SRC_DIR}/metrics ${CMAKE_CURRENT_BINARY_DIR}/metrics) -add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels) -add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes) -add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework) -add_subdirectory(${SRC_DIR}/engines ${CMAKE_CURRENT_BINARY_DIR}/engines) -add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output) - set(ENTITY ${PROJECT_NAME}.xc) set(SOURCES ${SRC_DIR}/entity.cpp) diff --git a/src/archetypes/energy_dist.h b/src/archetypes/energy_dist.h index 56a797751..a493d7d85 100644 --- a/src/archetypes/energy_dist.h +++ b/src/archetypes/energy_dist.h @@ -26,6 +26,8 @@ #include "utils/error.h" #include "utils/numeric.h" +#include "metrics/traits.h" + #include #include @@ -33,10 +35,9 @@ namespace arch { using namespace ntt; template + requires metric::traits::HasD struct EnergyDistribution { static constexpr auto D = M::Dim; - static constexpr bool is_energy_dist { true }; - static_assert(M::is_metric, "M must be a metric class"); EnergyDistribution(const M& metric) : metric { metric } {} @@ -249,7 +250,7 @@ namespace arch { } } - Inline void operator()(const coord_t& x_Code, vec_t& v) const { + Inline void operator()(const coord_t&, vec_t& v) const { if (cmp::AlmostZero(temperature)) { v[0] = ZERO; v[1] = ZERO; diff --git a/src/archetypes/field_setter.h b/src/archetypes/field_setter.h index 5c5c4dbe4..a9e45d2d6 100644 --- a/src/archetypes/field_setter.h +++ b/src/archetypes/field_setter.h @@ -27,33 +27,34 @@ #include "arch/traits.h" #include "utils/numeric.h" +#include "metrics/traits.h" + #include namespace arch { using namespace ntt; template + requires metric::traits::HasD && + (((S == SimEngine::SRPIC) && metric::traits::HasConvert && + metric::traits::HasTransform_i) || + ((S == SimEngine::GRPIC) && metric::traits::HasConvert_i)) && + (((S == SimEngine::SRPIC) && + (::traits::fieldsetter::HasEx1 || + ::traits::fieldsetter::HasEx2 || + ::traits::fieldsetter::HasEx3 || + ::traits::fieldsetter::HasBx1 || + ::traits::fieldsetter::HasBx2 || + ::traits::fieldsetter::HasBx3)) || + ((S == SimEngine::GRPIC) && + ((::traits::fieldsetter::HasDx1 && + ::traits::fieldsetter::HasDx2 && + ::traits::fieldsetter::HasDx3) || + (::traits::fieldsetter::HasBx1 && + ::traits::fieldsetter::HasBx2 && + ::traits::fieldsetter::HasBx3)))) class SetEMFields_kernel { static constexpr Dimension D = M::Dim; - static constexpr bool defines_ex1 = traits::has_method::value; - static constexpr bool defines_ex2 = traits::has_method::value; - static constexpr bool defines_ex3 = traits::has_method::value; - static constexpr bool defines_bx1 = traits::has_method::value; - static constexpr bool defines_bx2 = traits::has_method::value; - static constexpr bool defines_bx3 = traits::has_method::value; - static constexpr bool defines_dx1 = traits::has_method::value; - static constexpr bool defines_dx2 = traits::has_method::value; - static constexpr bool defines_dx3 = traits::has_method::value; - - static_assert(defines_ex1 || defines_ex2 || defines_ex3 || defines_bx1 || - defines_bx2 || defines_bx3 || defines_dx1 || defines_dx2 || - defines_dx3, - "No field initializer defined"); - static_assert((S != SimEngine::GRPIC) || - (defines_dx1 == defines_dx2 && defines_dx2 == defines_dx3 && - defines_bx1 == defines_bx2 && defines_bx2 == defines_bx3), - "In GR mode, all components must be defined or none"); - static_assert(M::is_metric, "M must be a metric class"); ndfield_t EM; const I finit; @@ -72,37 +73,37 @@ namespace arch { const auto i1_ = COORD(i1); coord_t x_Phys { ZERO }; if constexpr (S == SimEngine::SRPIC) { - if constexpr (defines_ex1) { + if constexpr (::traits::fieldsetter::HasEx1) { metric.template convert({ i1_ + HALF }, x_Phys); EM(i1, em::ex1) = metric.template transform<1, Idx::T, Idx::U>( { i1_ + HALF }, finit.ex1(x_Phys)); } - if constexpr (defines_ex2) { + if constexpr (::traits::fieldsetter::HasEx2) { metric.template convert({ i1_ }, x_Phys); EM(i1, em::ex2) = metric.template transform<2, Idx::T, Idx::U>( { i1_ }, finit.ex2(x_Phys)); } - if constexpr (defines_ex3) { + if constexpr (::traits::fieldsetter::HasEx3) { metric.template convert({ i1_ }, x_Phys); EM(i1, em::ex3) = metric.template transform<3, Idx::T, Idx::U>( { i1_ }, finit.ex3(x_Phys)); } - if constexpr (defines_bx1) { + if constexpr (::traits::fieldsetter::HasBx1) { metric.template convert({ i1_ }, x_Phys); EM(i1, em::bx1) = metric.template transform<1, Idx::T, Idx::U>( { i1_ }, finit.bx1(x_Phys)); } - if constexpr (defines_bx2) { + if constexpr (::traits::fieldsetter::HasBx2) { metric.template convert({ i1_ + HALF }, x_Phys); EM(i1, em::bx2) = metric.template transform<2, Idx::T, Idx::U>( { i1_ + HALF }, finit.bx2(x_Phys)); } - if constexpr (defines_bx3) { + if constexpr (::traits::fieldsetter::HasBx3) { metric.template convert({ i1_ + HALF }, x_Phys); EM(i1, em::bx3) = metric.template transform<3, Idx::T, Idx::U>( { i1_ + HALF }, @@ -123,37 +124,37 @@ namespace arch { // srpic if constexpr (S == SimEngine::SRPIC) { coord_t x_Phys { ZERO }; - if constexpr (defines_ex1) { + if constexpr (::traits::fieldsetter::HasEx1) { metric.template convert({ i1_ + HALF, i2_ }, x_Phys); EM(i1, i2, em::ex1) = metric.template transform<1, Idx::T, Idx::U>( { i1_ + HALF, i2_ }, finit.ex1(x_Phys)); } - if constexpr (defines_ex2) { + if constexpr (::traits::fieldsetter::HasEx2) { metric.template convert({ i1_, i2_ + HALF }, x_Phys); EM(i1, i2, em::ex2) = metric.template transform<2, Idx::T, Idx::U>( { i1_, i2_ + HALF }, finit.ex2(x_Phys)); } - if constexpr (defines_ex3) { + if constexpr (::traits::fieldsetter::HasEx3) { metric.template convert({ i1_, i2_ }, x_Phys); EM(i1, i2, em::ex3) = metric.template transform<3, Idx::T, Idx::U>( { i1_, i2_ }, finit.ex3(x_Phys)); } - if constexpr (defines_bx1) { + if constexpr (::traits::fieldsetter::HasBx1) { metric.template convert({ i1_, i2_ + HALF }, x_Phys); EM(i1, i2, em::bx1) = metric.template transform<1, Idx::T, Idx::U>( { i1_, i2_ + HALF }, finit.bx1(x_Phys)); } - if constexpr (defines_bx2) { + if constexpr (::traits::fieldsetter::HasBx2) { metric.template convert({ i1_ + HALF, i2_ }, x_Phys); EM(i1, i2, em::bx2) = metric.template transform<2, Idx::T, Idx::U>( { i1_ + HALF, i2_ }, finit.bx2(x_Phys)); } - if constexpr (defines_bx3) { + if constexpr (::traits::fieldsetter::HasBx3) { metric.template convert({ i1_ + HALF, i2_ + HALF }, x_Phys); EM(i1, i2, em::bx3) = metric.template transform<3, Idx::T, Idx::U>( @@ -162,7 +163,9 @@ namespace arch { } } else if constexpr (S == SimEngine::GRPIC) { // grpic - if constexpr (defines_dx1 && defines_dx2 && defines_dx3) { + if constexpr (::traits::fieldsetter::HasDx1 && + ::traits::fieldsetter::HasDx2 && + ::traits::fieldsetter::HasDx3) { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; @@ -179,7 +182,9 @@ namespace arch { EM(i1, i2, em::dx3) = finit.dx3({ x1_0, x2_0 }); } } - if constexpr (defines_bx1 && defines_bx2 && defines_bx3) { + if constexpr (::traits::fieldsetter::HasBx1 && + ::traits::fieldsetter::HasBx2 && + ::traits::fieldsetter::HasBx3) { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( i1_ + HALF) }; @@ -212,28 +217,28 @@ namespace arch { coord_t x_Phys { ZERO }; if constexpr (S == SimEngine::SRPIC) { // srpic - if constexpr (defines_ex1) { + if constexpr (::traits::fieldsetter::HasEx1) { metric.template convert({ i1_ + HALF, i2_, i3_ }, x_Phys); EM(i1, i2, i3, em::ex1) = metric.template transform<1, Idx::T, Idx::U>( { i1_ + HALF, i2_, i3_ }, finit.ex1(x_Phys)); } - if constexpr (defines_ex2) { + if constexpr (::traits::fieldsetter::HasEx2) { metric.template convert({ i1_, i2_ + HALF, i3_ }, x_Phys); EM(i1, i2, i3, em::ex2) = metric.template transform<2, Idx::T, Idx::U>( { i1_, i2_ + HALF, i3_ }, finit.ex2(x_Phys)); } - if constexpr (defines_ex3) { + if constexpr (::traits::fieldsetter::HasEx3) { metric.template convert({ i1_, i2_, i3_ + HALF }, x_Phys); EM(i1, i2, i3, em::ex3) = metric.template transform<3, Idx::T, Idx::U>( { i1_, i2_, i3_ + HALF }, finit.ex3(x_Phys)); } - if constexpr (defines_bx1) { + if constexpr (::traits::fieldsetter::HasBx1) { metric.template convert( { i1_, i2_ + HALF, i3_ + HALF }, x_Phys); @@ -241,7 +246,7 @@ namespace arch { { i1_, i2_ + HALF, i3_ + HALF }, finit.bx1(x_Phys)); } - if constexpr (defines_bx2) { + if constexpr (::traits::fieldsetter::HasBx2) { metric.template convert( { i1_ + HALF, i2_, i3_ + HALF }, x_Phys); @@ -249,7 +254,7 @@ namespace arch { { i1_ + HALF, i2_, i3_ + HALF }, finit.bx2(x_Phys)); } - if constexpr (defines_bx3) { + if constexpr (::traits::fieldsetter::HasBx3) { metric.template convert( { i1_ + HALF, i2_ + HALF, i3_ }, x_Phys); @@ -269,71 +274,30 @@ namespace arch { const real_t x3_H { metric.template convert<3, Crd::Cd, Crd::Ph>( i3_ + HALF) }; - if constexpr (defines_dx1 && defines_dx2 && defines_dx3) { + if constexpr (::traits::fieldsetter::HasDx1 && + ::traits::fieldsetter::HasDx2 && + ::traits::fieldsetter::HasDx3) { { // dx1 - vec_t d_PU { finit.dx1({ x1_H, x2_0, x3_0 }), - finit.dx2({ x1_H, x2_0, x3_0 }), - finit.dx3({ x1_H, x2_0, x3_0 }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_ + HALF, i2_, i3_ }, - d_PU, - d_U); - EM(i1, i2, i3, em::dx1) = d_U[0]; + EM(i1, i2, i3, em::dx1) = finit.dx1({ x1_H, x2_0, x3_0 }); } { // dx2 - vec_t d_PU { finit.dx1({ x1_0, x2_H, x3_0 }), - finit.dx2({ x1_0, x2_H, x3_0 }), - finit.dx3({ x1_0, x2_H, x3_0 }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_, i2_ + HALF, i3_ }, - d_PU, - d_U); - EM(i1, i2, i3, em::dx2) = d_U[1]; + EM(i1, i2, i3, em::dx2) = finit.dx2({ x1_0, x2_H, x3_0 }); } { // dx3 - vec_t d_PU { finit.dx1({ x1_0, x2_0, x3_H }), - finit.dx2({ x1_0, x2_0, x3_H }), - finit.dx3({ x1_0, x2_0, x3_H }) }; - vec_t d_U { ZERO }; - metric.template transform({ i1_, i2_, i3_ + HALF }, - d_PU, - d_U); - EM(i1, i2, i3, em::dx3) = d_U[2]; + EM(i1, i2, i3, em::dx3) = finit.dx3({ x1_0, x2_0, x3_H }); } } - if constexpr (defines_bx1 && defines_bx2 && defines_bx3) { + if constexpr (::traits::fieldsetter::HasBx1 && + ::traits::fieldsetter::HasBx2 && + ::traits::fieldsetter::HasBx3) { { // bx1 - vec_t b_PU { finit.bx1({ x1_0, x2_H, x3_H }), - finit.bx2({ x1_0, x2_H, x3_H }), - finit.bx3({ x1_0, x2_H, x3_H }) }; - vec_t b_U { ZERO }; - metric.template transform( - { i1_, i2_ + HALF, i3_ + HALF }, - b_PU, - b_U); - EM(i1, i2, i3, em::bx1) = b_U[0]; + EM(i1, i2, i3, em::bx1) = finit.bx1({ x1_0, x2_H, x3_H }); } { // bx2 - vec_t b_PU { finit.bx1({ x1_H, x2_0, x3_H }), - finit.bx2({ x1_H, x2_0, x3_H }), - finit.bx3({ x1_H, x2_0, x3_H }) }; - vec_t b_U { ZERO }; - metric.template transform( - { i1_ + HALF, i2_, i3_ + HALF }, - b_PU, - b_U); - EM(i1, i2, i3, em::bx2) = b_U[1]; + EM(i1, i2, i3, em::bx2) = finit.bx2({ x1_H, x2_0, x3_H }); } { // bx3 - vec_t b_PU { finit.bx1({ x1_H, x2_H, x3_0 }), - finit.bx2({ x1_H, x2_H, x3_0 }), - finit.bx3({ x1_H, x2_H, x3_0 }) }; - vec_t b_U { ZERO }; - metric.template transform( - { i1_ + HALF, i2_ + HALF, i3_ }, - b_PU, - b_U); - EM(i1, i2, i3, em::bx3) = b_U[2]; + EM(i1, i2, i3, em::bx3) = finit.bx3({ x1_H, x2_H, x3_0 }); } } } else { diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 24af59965..e1ca1b7ca 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -22,9 +22,10 @@ #include "utils/error.h" #include "utils/numeric.h" +#include "metrics/traits.h" + #include "framework/domain/domain.h" #include "framework/domain/metadomain.h" - #include "kernels/injectors.hpp" #include @@ -53,6 +54,7 @@ namespace arch { * - array_t: maximum coordinates of the region in computational coords */ template + requires metric::traits::HasD && metric::traits::HasConvert auto DeduceRegion(const Domain& domain, const boundaries_t& box) -> std::tuple, array_t> { if (not domain.mesh.Intersects(box)) { @@ -107,6 +109,7 @@ namespace arch { * - array_t: maximum coordinates of the region in computational coords */ template + requires metric::traits::HasD auto ComputeNumInject(const SimulationParams& params, const Domain& domain, real_t number_density, @@ -198,6 +201,8 @@ namespace arch { * @tparam ED2 Energy distribution type for species 2 */ template + requires metric::traits::HasD && traits::energydist::IsValid && + traits::energydist::IsValid inline void InjectUniform(const SimulationParams& params, Domain& domain, const std::pair& species, @@ -205,9 +210,6 @@ namespace arch { real_t number_density, bool use_weights = false, const boundaries_t& box = {}) { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(ED1::is_energy_dist, "ED1 must be an energy distribution class"); - static_assert(ED2::is_energy_dist, "ED2 must be an energy distribution class"); raise::ErrorIf((M::CoordType != Coord::Cart) && (not use_weights), "Weights must be used for non-Cartesian coordinates", HERE); @@ -278,7 +280,6 @@ namespace arch { spidx_t spidx, const std::map>& data, bool use_weights = false) { - static_assert(M::is_metric, "M must be a metric class"); const auto n_inject = data.at("ux1").size(); auto injector_kernel = kernel::GlobalInjector_kernel( local_domain.species[spidx - 1], @@ -309,6 +310,8 @@ namespace arch { * @tparam SD Spatial distribution type */ template + requires metric::traits::HasD && traits::energydist::IsValid && + traits::energydist::IsValid && traits::spatialdist::IsValid inline void InjectNonUniform(const SimulationParams& params, Domain& domain, const std::pair& species, @@ -317,10 +320,6 @@ namespace arch { real_t number_density, bool use_weights = false, const boundaries_t& box = {}) { - static_assert(M::is_metric, "M must be a metric class"); - static_assert(ED1::is_energy_dist, "ED1 must be an energy distribution class"); - static_assert(ED2::is_energy_dist, "ED2 must be an energy distribution class"); - static_assert(SD::is_spatial_dist, "SD must be a spatial distribution class"); raise::ErrorIf((M::CoordType != Coord::Cart) && (not use_weights), "Weights must be used for non-Cartesian coordinates", HERE); diff --git a/src/archetypes/problem_generator.h b/src/archetypes/problem_generator.h index da116672d..fc22428e6 100644 --- a/src/archetypes/problem_generator.h +++ b/src/archetypes/problem_generator.h @@ -23,15 +23,16 @@ #include "enums.h" #include "global.h" -#include "framework/parameters.h" +#include "metrics/traits.h" + +#include "framework/parameters/parameters.h" namespace arch { using namespace ntt; template + requires metric::traits::HasD and metric::traits::HasCoordType struct ProblemGenerator { - static_assert(M::is_metric, "M must be a metric class"); - static constexpr bool is_pgen { true }; static constexpr Dimension D { M::Dim }; static constexpr Coord C { M::CoordType }; diff --git a/src/archetypes/spatial_dist.h b/src/archetypes/spatial_dist.h index 68477208c..bf8abe9cd 100644 --- a/src/archetypes/spatial_dist.h +++ b/src/archetypes/spatial_dist.h @@ -23,13 +23,15 @@ #include "utils/error.h" #include "utils/numeric.h" +#include "metrics/traits.h" + namespace arch { using namespace ntt; template + requires metric::traits::HasD struct SpatialDistribution { - static constexpr bool is_spatial_dist { true }; - static_assert(M::is_metric, "M must be a metric class"); + static constexpr auto D = M::Dim; SpatialDistribution(const M& metric) : metric { metric } {} diff --git a/src/archetypes/tests/CMakeLists.txt b/src/archetypes/tests/CMakeLists.txt index 9419847c5..4a5b501e1 100644 --- a/src/archetypes/tests/CMakeLists.txt +++ b/src/archetypes/tests/CMakeLists.txt @@ -27,3 +27,4 @@ gen_test(energy_dist) gen_test(spatial_dist) gen_test(field_setter) gen_test(powerlaw) +gen_test(pgen) diff --git a/src/archetypes/tests/pgen.cpp b/src/archetypes/tests/pgen.cpp new file mode 100644 index 000000000..3eba67611 --- /dev/null +++ b/src/archetypes/tests/pgen.cpp @@ -0,0 +1,185 @@ +#include "enums.h" + +#include "utils/numeric.h" + +#include "metrics/minkowski.h" + +#include "archetypes/problem_generator.h" +#include "archetypes/traits.h" +#include "framework/domain/domain.h" +#include "framework/parameters/parameters.h" + +#include + +#include + +using namespace ntt; + +template +struct CustomFieldsetter { + Inline auto ex1(const coord_t&) const -> real_t { + return ZERO; + } +}; + +template +struct ExtForce { + Inline auto fx1(const coord_t&) const -> real_t { + return ZERO; + } + + Inline auto ex1(const coord_t&) const -> real_t { + return ZERO; + } + + Inline auto bx3(const coord_t&) const -> real_t { + return ZERO; + } +}; + +template +struct ExtCurrent { + Inline auto jx1(const coord_t&) const -> real_t { + return ZERO; + } +}; + +template +struct CustomPgen : public arch::ProblemGenerator { + CustomPgen(const SimulationParams& params = {}) + : arch::ProblemGenerator { params } {} + + CustomFieldsetter init_flds {}; + ExtCurrent ext_current {}; + + void InitPrtls(Domain&) {} + + auto AtmFields(simtime_t) const -> CustomFieldsetter { + return init_flds; + } + + auto MatchFields(simtime_t) const -> CustomFieldsetter { + return init_flds; + } + + auto MatchFieldsInX1(simtime_t) const -> CustomFieldsetter { + return init_flds; + } + + auto MatchFieldsInX2(simtime_t) const -> CustomFieldsetter { + return init_flds; + } + + auto MatchFieldsInX3(simtime_t) const -> CustomFieldsetter { + return init_flds; + } + + auto ExternalFields(simtime_t, spidx_t, const Domain&) const + -> std::pair> { + return { true, ExtForce {} }; + } + + auto FixFieldsConst(simtime_t, const bc_in&, ntt::em) const + -> std::pair { + return { ZERO, false }; + } + + void CustomPostStep(timestep_t, simtime_t, Domain&) {} + + void CustomFieldOutput(const std::string&, + ndfield_t&, + index_t, + timestep_t, + simtime_t, + const Domain&) {} + + auto CustomStat(const std::string&, timestep_t, simtime_t, const Domain&) + -> real_t { + return ZERO; + } +}; + +auto main(int argc, char* argv[]) -> int { + Kokkos::initialize(argc, argv); + try { + auto custom_pgen = CustomPgen> {}; + + if constexpr (not arch::traits::pgen::HasInitFlds) { + throw std::runtime_error("CustomPgen should have init_flds"); + } + if constexpr (not arch::traits::pgen::HasInitPrtls< + decltype(custom_pgen), + Domain>>) { + throw std::runtime_error("CustomPgen should have InitPrtls"); + } + if constexpr (not arch::traits::pgen::HasExternalFields< + decltype(custom_pgen), + Domain>>) { + throw std::runtime_error("CustomPgen should have ext_fields"); + } + if constexpr (not arch::traits::pgen::HasExtCurrent) { + throw std::runtime_error("CustomPgen should have ext_current"); + } + if constexpr (not arch::traits::pgen::HasAtmFields) { + throw std::runtime_error("CustomPgen should have AtmFields"); + } + if constexpr (not arch::traits::pgen::HasMatchFields) { + throw std::runtime_error("CustomPgen should have MatchFields"); + } + if constexpr ( + not arch::traits::pgen::HasMatchFieldsInX1) { + throw std::runtime_error("CustomPgen should have MatchFieldsInX1"); + } + if constexpr ( + not arch::traits::pgen::HasMatchFieldsInX2) { + throw std::runtime_error("CustomPgen should have MatchFieldsInX2"); + } + if constexpr ( + not arch::traits::pgen::HasMatchFieldsInX3) { + throw std::runtime_error("CustomPgen should have MatchFieldsInX3"); + } + if constexpr (not arch::traits::pgen::HasFixFieldsConst) { + throw std::runtime_error("CustomPgen should have FixFieldsConst"); + } + if constexpr (not arch::traits::pgen::HasCustomPostStep< + decltype(custom_pgen), + Domain>>) { + throw std::runtime_error("CustomPgen should have CustomPostStep"); + } + if constexpr (not arch::traits::pgen::HasCustomFieldOutput< + decltype(custom_pgen), + Domain>>) { + throw std::runtime_error("CustomPgen should have CustomFieldOutput"); + } + if constexpr (not arch::traits::pgen::HasCustomStatOutput< + decltype(custom_pgen), + Domain>>) { + throw std::runtime_error("CustomPgen should have CustomStat"); + } + auto domain = Domain> { + false, 0u, { 0u }, { 0u }, { 10u }, { { (real_t)0.0, (real_t)1.0 } }, + {}, {} + }; + auto [apply_extfields, + ext_fields] = custom_pgen.ExternalFields(ZERO, 0, domain); + if constexpr (not ::traits::external::HasFx1) { + throw std::runtime_error("CustomPgen's ext_fields should have fx1"); + } + if constexpr (not ::traits::external::HasEx1) { + throw std::runtime_error("CustomPgen's ext_fields should have ex1"); + } + if constexpr (::traits::external::HasBx1) { + throw std::runtime_error("CustomPgen's ext_fields should not have bx1"); + } + if constexpr (not ::traits::external::HasBx3) { + throw std::runtime_error("CustomPgen's ext_current should have bx3"); + } + + } catch (std::exception& e) { + std::cerr << e.what() << std::endl; + Kokkos::finalize(); + return 1; + } + Kokkos::finalize(); + return 0; +} diff --git a/src/archetypes/traits.h b/src/archetypes/traits.h new file mode 100644 index 000000000..f453c7fc0 --- /dev/null +++ b/src/archetypes/traits.h @@ -0,0 +1,175 @@ +/** + * @file archetypes/traits.h + * @brief Defines a set of traits to check if archetype classes satisfy certain conditions + * @implements + * - arch::traits::energydist::IsValid<> - checks if energy distribution class has required operator() + * - arch::traits::spatialdist::IsValid<> - checks if spatial distribution class has required operator() + * - arch::traits::pgen::check_compatibility<> - checks if problem generator is compatible with given enums + * - arch::traits::pgen::compatible_with<> - defines compatible enums for problem generator + * - arch::traits::pgen::HasD<> - checks if problem generator has Dim static member + * - arch::traits::pgen::HasInitFlds<> - checks if problem generator has init_flds member + * - arch::traits::pgen::HasInitPrtls<> - checks if problem generator has InitPrtls method + * - arch::traits::pgen::HasExternalFields<> - checks if problem generator has ExternalFields method + * - arch::traits::pgen::HasExtCurrent<> - checks if problem generator has ext_current member + * - arch::traits::pgen::HasAtmFields<> - checks if problem generator has AtmFields method + * - arch::traits::pgen::HasMatchFields<> - checks if problem generator has MatchFields method + * - arch::traits::pgen::HasMatchFieldsInX1<> - checks if problem generator has MatchFieldsInX1 method + * - arch::traits::pgen::HasMatchFieldsInX2<> - checks if problem generator has MatchFieldsInX2 method + * - arch::traits::pgen::HasMatchFieldsInX3<> - checks if problem generator has MatchFieldsInX3 method + * - arch::traits::pgen::HasFixFieldsConst<> - checks if problem generator has FixFieldsConst method + * - arch::traits::pgen::HasCustomPostStep<> - checks if problem generator has CustomPostStep method + * - arch::traits::pgen::HasCustomFieldOutput<> - checks if problem generator has CustomFieldOutput method + * - arch::traits::pgen::HasCustomStatOutput<> - checks if problem generator has CustomStat method + * @namespaces: + * - arch::traits:: + */ +#ifndef ARCHETYPES_TRAITS_H +#define ARCHETYPES_TRAITS_H + +#include "global.h" + +#include "arch/kokkos_aliases.h" + +namespace arch { + namespace traits { + + namespace energydist { + + template + concept IsValid = requires(const ED& edist, + const coord_t& x_Ph, + vec_t& v) { + { edist(x_Ph, v) } -> std::same_as; + }; + + } // namespace energydist + + namespace spatialdist { + + template + concept IsValid = requires(const SD& sdist, const coord_t& x_Ph) { + { sdist(x_Ph) } -> std::convertible_to; + }; + + } // namespace spatialdist + + namespace pgen { + + // checking compat for the problem generator + engine + template + struct check_compatibility { + template + static constexpr bool value(std::integer_sequence) { + return ((Is == N) || ...); + } + }; + + template + struct compatible_with { + static constexpr auto value = std::integer_sequence {}; + }; + + template + concept HasD = requires { + { PG::D } -> std::convertible_to; + }; + + template + concept HasInitFlds = requires(const PG& pgen) { pgen.init_flds; }; + + template + concept HasEmissionPolicy = requires(const PG& pgen, + simtime_t time, + spidx_t sp, + D& domain) { + pgen.EmissionPolicy(time, sp, domain); + }; + + template + concept HasInitPrtls = requires(PG& pgen, D& domain) { + { pgen.InitPrtls(domain) } -> std::same_as; + }; + + template + concept HasExternalFields = requires(const PG& pgen, + simtime_t time, + spidx_t sp, + D& domain) { + requires std::same_as; + pgen.ExternalFields(time, sp, domain).second; + }; + + template + concept HasExtCurrent = requires(const PG& pgen) { pgen.ext_current; }; + + template + concept HasAtmFields = requires(const PG& pgen, simtime_t time) { + pgen.AtmFields(time); + }; + + template + concept HasMatchFields = requires(const PG& pgen, simtime_t time) { + pgen.MatchFields(time); + }; + + template + concept HasMatchFieldsInX1 = requires(const PG& pgen, simtime_t time) { + pgen.MatchFieldsInX1(time); + }; + + template + concept HasMatchFieldsInX2 = requires(const PG& pgen, simtime_t time) { + pgen.MatchFieldsInX2(time); + }; + + template + concept HasMatchFieldsInX3 = requires(const PG& pgen, simtime_t time) { + pgen.MatchFieldsInX3(time); + }; + + template + concept HasFixFieldsConst = requires(const PG& pgen, + simtime_t time, + const bc_in& bc, + ntt::em comp) { + { + pgen.FixFieldsConst(time, bc, comp) + } -> std::convertible_to>; + }; + + template + concept HasCustomPostStep = requires(PG& pgen, + timestep_t s, + simtime_t t, + D& domain) { + { pgen.CustomPostStep(s, t, domain) } -> std::same_as; + }; + + template + concept HasCustomFieldOutput = requires(PG& pgen, + const std::string& name, + ndfield_t& buff, + index_t idx, + timestep_t step, + simtime_t time, + const D& dom) { + { + pgen.CustomFieldOutput(name, buff, idx, step, time, dom) + } -> std::same_as; + }; + + template + concept HasCustomStatOutput = requires(PG& pgen, + const std::string& name, + timestep_t s, + simtime_t t, + const D& dom) { + { pgen.CustomStat(name, s, t, dom) } -> std::convertible_to; + }; + + } // namespace pgen + } // namespace traits +} // namespace arch + +#endif // ARCHETYPES_TRAITS_H diff --git a/src/archetypes/utils.h b/src/archetypes/utils.h index b1a29cb71..14ac7d8f8 100644 --- a/src/archetypes/utils.h +++ b/src/archetypes/utils.h @@ -17,7 +17,7 @@ #include "archetypes/energy_dist.h" #include "archetypes/particle_injector.h" #include "framework/domain/domain.h" -#include "framework/parameters.h" +#include "framework/parameters/parameters.h" #include @@ -45,7 +45,6 @@ namespace arch { const std::pair, std::vector>& drift_four_vels = {{ ZERO, ZERO, ZERO }, { ZERO, ZERO, ZERO }}, bool use_weights = false, const boundaries_t& box = {}) { - static_assert(M::is_metric, "M must be a metric class"); const auto mass_1 = domain.species[species.first - 1].mass(); const auto mass_2 = domain.species[species.second - 1].mass(); @@ -93,7 +92,6 @@ namespace arch { const std::pair, std::vector>& drift_four_vels = {{ ZERO, ZERO, ZERO }, { ZERO, ZERO, ZERO }}, bool use_weights = false, const boundaries_t& box = {}) { - static_assert(M::is_metric, "M must be a metric class"); InjectUniformMaxwellians(params, domain, diff --git a/src/engines/CMakeLists.txt b/src/engines/CMakeLists.txt index 4cef18630..5b2edebdc 100644 --- a/src/engines/CMakeLists.txt +++ b/src/engines/CMakeLists.txt @@ -4,9 +4,7 @@ # # @sources: # -# * engine_printer.cpp -# * engine_init.cpp -# * engine_run.cpp +# * reporter.cpp # # @includes: # @@ -32,8 +30,7 @@ # ------------------------------ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SOURCES ${SRC_DIR}/engine_printer.cpp ${SRC_DIR}/engine_init.cpp - ${SRC_DIR}/engine_run.cpp) +set(SOURCES ${SRC_DIR}/reporter.cpp) add_library(ntt_engines ${SOURCES}) set(libs ntt_global ntt_framework ntt_metrics ntt_archetypes ntt_kernels @@ -43,7 +40,7 @@ if(${output}) endif() add_dependencies(ntt_engines ${libs}) target_link_libraries(ntt_engines PUBLIC ${libs}) -target_compile_definitions(ntt_engines PRIVATE PGEN=\"${PGEN}\") +target_compile_definitions(ntt_engines PUBLIC PGEN=\"${PGEN}\") target_include_directories( ntt_engines diff --git a/src/engines/engine.hpp b/src/engines/engine.hpp index 17103f1de..0d0948e79 100644 --- a/src/engines/engine.hpp +++ b/src/engines/engine.hpp @@ -3,16 +3,11 @@ * @brief Base simulation class which just initializes the metadomain * @implements * - ntt::Engine<> - * @cpp: - * - engine_init.cpp - * - engine_printer.cpp * @namespaces: * - ntt:: * @macros: - * - MPI - * - DEBUG + * - MPI_ENABLED * - OUTPUT_ENABLED - * - CUDA_ENABLED */ #ifndef ENGINES_ENGINE_H @@ -21,22 +16,36 @@ #include "enums.h" #include "global.h" -#include "arch/traits.h" -#include "utils/error.h" +#include "arch/mpi_aliases.h" +#include "utils/diag.h" +#include "utils/reporter.h" #include "utils/timer.h" -#include "utils/toml.h" +#include "archetypes/field_setter.h" +#include "archetypes/traits.h" +#include "engines/reporter.h" +#include "engines/traits.h" #include "framework/containers/species.h" +#include "framework/domain/domain.h" #include "framework/domain/metadomain.h" -#include "framework/parameters.h" +#include "framework/parameters/parameters.h" #include "pgen.hpp" +#include + +#if defined(OUTPUT_ENABLED) + #include +#endif + #include +#include +#include + #if defined(OUTPUT_ENABLED) #include - #include + #include #endif // OUTPUT_ENABLED #if defined(MPI_ENABLED) @@ -49,9 +58,8 @@ namespace ntt { template + requires traits::engine::IsCompatibleWithEngine class Engine { - static_assert(M::is_metric, "template arg for Engine class has to be a metric"); - static_assert(user::PGen::is_pgen, "unrecognized problem generator"); protected: #if defined(OUTPUT_ENABLED) @@ -76,14 +84,7 @@ namespace ntt { timestep_t step; public: - static constexpr bool pgen_is_ok { - traits::check_compatibility::value(user::PGen::engines) and - traits::check_compatibility::value(user::PGen::metrics) and - traits::check_compatibility::value(user::PGen::dimensions) - }; - static constexpr Dimension D { M::Dim }; - static constexpr bool is_engine { true }; Engine(const SimulationParams& params) : m_params { params } @@ -108,9 +109,7 @@ namespace ntt { , start_step { m_params.get("checkpoint.start_step") } , start_time { m_params.get("checkpoint.start_time") } , time { start_time } - , step { start_step } { - raise::ErrorIf(not pgen_is_ok, "Problem generator is not compatible with the picked engine/metric/dimension", HERE); - } + , step { start_step } {} ~Engine() = default; @@ -120,8 +119,250 @@ namespace ntt { virtual void step_forward(timer::Timers&, Domain&) = 0; void run(); + + auto engineParams() const -> prm::Parameters { + auto parameters = prm::Parameters {}; + parameters.set("dt", static_cast(dt)); + parameters.set("time", static_cast(time)); + return parameters; + } }; + template + requires traits::engine::IsCompatibleWithEngine + void Engine::init() { + m_metadomain.InitStatsWriter(m_params, is_resuming); +#if defined(OUTPUT_ENABLED) + m_metadomain.InitWriter(&m_adios, m_params); + m_metadomain.InitCheckpointWriter(&m_adios, m_params); +#endif + logger::Checkpoint("Initializing Engine", HERE); + if (not is_resuming) { + // start a new simulation with initial conditions + logger::Checkpoint("Loading initial conditions", HERE); + if constexpr (arch::traits::pgen::HasInitFlds>) { + logger::Checkpoint("Initializing fields from problem generator", HERE); + m_metadomain.runOnLocalDomains([&](auto& loc_dom) { + Kokkos::parallel_for( + "InitFields", + loc_dom.mesh.rangeActiveCells(), + arch::SetEMFields_kernel { + loc_dom.fields.em, + m_pgen.init_flds, + loc_dom.mesh.metric }); + }); + } + if constexpr ( + arch::traits::pgen::HasInitPrtls, Domain>) { + logger::Checkpoint("Initializing particles from problem generator", HERE); + m_metadomain.runOnLocalDomains([&](auto& loc_dom) { + m_pgen.InitPrtls(loc_dom); + }); + } + } else { +#if defined(OUTPUT_ENABLED) + // read simulation data from the checkpoint + raise::ErrorIf( + m_params.template get("checkpoint.start_step") == 0, + "Resuming simulation from a checkpoint requires a valid start_step", + HERE); + logger::Checkpoint("Resuming simulation from a checkpoint", HERE); + m_metadomain.ContinueFromCheckpoint(&m_adios, m_params); +#else + raise::Error( + "Resuming simulation from a checkpoint requires -D output=ON", + HERE); +#endif + } + print_report(); + } + + template + requires traits::engine::IsCompatibleWithEngine + void Engine::print_report() const { + const auto colored_stdout = m_params.template get( + "diagnostics.colored_stdout"); + std::string report = ""; + CallOnce( + [&](auto& metadomain, auto& params) { + report += reporter::Backend(); + + report += ReportSimulationConfig(params, + S, + Metric(M::MetricType), + dt, + runtime, + max_steps, + metadomain.ndomains_per_dim(), + metadomain.ndomains()); + const auto pgen_name = std::string(PGEN); + report += ReportPgenConfig>(m_pgen, + pgen_name); + if (metadomain.species_params().size() > 0) { + report += "\n"; + reporter::AddCategory(report, 4, "Particles"); + } + for (const auto& species : metadomain.species_params()) { + report += species.Report(); + } + report.pop_back(); + }, + m_metadomain, + m_params); + info::Print(report, colored_stdout); + + report = "\n"; + CallOnce([&]() { + reporter::AddCategory(report, 4, "Domains"); + report.pop_back(); + }); + info::Print(report, colored_stdout); + + for (unsigned int idx { 0 }; idx < m_metadomain.ndomains(); ++idx) { + auto is_local = false; + for (const auto& lidx : m_metadomain.l_subdomain_indices()) { + is_local |= (idx == lidx); + } + if (is_local) { + const auto& domain = m_metadomain.subdomain(idx); + report = domain.Report(); + if (idx == m_metadomain.ndomains() - 1) { + report += "\n\n"; + } + info::Print(report, colored_stdout, true, false); + } +#if defined(MPI_ENABLED) + MPI_Barrier(MPI_COMM_WORLD); +#endif + } + } + + template + requires traits::engine::IsCompatibleWithEngine + void Engine::run() { + init(); + + auto timers = timer::Timers { + { "FieldSolver", + "CurrentFiltering", "CurrentDeposit", + "ParticlePusher", "FieldBoundaries", + "ParticleBoundaries", "Communications", + "Injector", "Custom", + "ParticleSort", "Output", + "Checkpoint" }, + []() { + Kokkos::fence(); + }, + m_params.get("diagnostics.blocking_timers") + }; + const auto diag_interval = m_params.template get( + "diagnostics.interval"); + + auto time_history = pbar::DurationHistory { 1000 }; + const auto clear_interval = m_params.template get( + "particles.clear_interval"); + + // main algorithm loop + while (step < max_steps) { + // run the engine-dependent algorithm step + m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { + step_forward(timers, dom); + }); + // poststep (if defined) + if constexpr ( + arch::traits::pgen::HasCustomPostStep>) { + timers.start("Custom"); + m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { + m_pgen.CustomPostStep(step, time, dom); + }); + timers.stop("Custom"); + } + auto print_prtl_clear = (clear_interval > 0 and + step % clear_interval == 0 and step > 0); + + // advance time & step + time += dt; + ++step; + + auto print_output = false; + auto print_checkpoint = false; +#if defined(OUTPUT_ENABLED) + timers.start("Output"); + if constexpr ( + arch::traits::pgen::HasCustomFieldOutput>) { + auto lambda_custom_field_output = [&](const std::string& name, + ndfield_t& buff, + index_t idx, + timestep_t step, + simtime_t time, + const Domain& dom) { + m_pgen.CustomFieldOutput(name, buff, idx, step, time, dom); + }; + print_output &= m_metadomain.Write(m_params, + step, + step - 1, + time, + time - dt, + lambda_custom_field_output); + } else { + print_output &= m_metadomain.Write(m_params, step, step - 1, time, time - dt); + } + if constexpr ( + arch::traits::pgen::HasCustomStatOutput>) { + auto lambda_custom_stat = [&](const std::string& name, + timestep_t step, + simtime_t time, + const Domain& dom) -> real_t { + return m_pgen.CustomStat(name, step, time, dom); + }; + print_output &= m_metadomain.WriteStats(m_params, + step, + step - 1, + time, + time - dt, + lambda_custom_stat); + } else { + print_output &= m_metadomain.WriteStats(m_params, + step, + step - 1, + time, + time - dt); + } + timers.stop("Output"); + + timers.start("Checkpoint"); + print_checkpoint = m_metadomain.WriteCheckpoint(m_params, + step, + step - 1, + time, + time - dt); + timers.stop("Checkpoint"); +#endif + + // advance time_history + time_history.tick(); + // print timestep report + if (diag_interval > 0 and step % diag_interval == 0) { + diag::printDiagnostics( + step - 1, + max_steps, + time - dt, + dt, + timers, + time_history, + m_metadomain.l_ncells(), + m_metadomain.species_labels(), + m_metadomain.l_npart_perspec(), + m_metadomain.l_maxnpart_perspec(), + print_prtl_clear, + print_output, + print_checkpoint, + m_params.get("diagnostics.colored_stdout")); + } + timers.resetAll(); + } + } + } // namespace ntt #endif // ENGINES_ENGINE_H diff --git a/src/engines/engine_init.cpp b/src/engines/engine_init.cpp deleted file mode 100644 index f98b117dd..000000000 --- a/src/engines/engine_init.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include "enums.h" -#include "global.h" - -#include "arch/traits.h" - -#include "archetypes/field_setter.h" -#include "framework/specialization_registry.h" - -#include "engines/engine.hpp" - -#include - -#include - -namespace ntt { - - template - void Engine::init() { - if constexpr (pgen_is_ok) { - m_metadomain.InitStatsWriter(m_params, is_resuming); -#if defined(OUTPUT_ENABLED) - m_metadomain.InitWriter(&m_adios, m_params); - m_metadomain.InitCheckpointWriter(&m_adios, m_params); -#endif - logger::Checkpoint("Initializing Engine", HERE); - if (not is_resuming) { - // start a new simulation with initial conditions - logger::Checkpoint("Loading initial conditions", HERE); - if constexpr ( - traits::has_member>::value) { - logger::Checkpoint("Initializing fields from problem generator", HERE); - m_metadomain.runOnLocalDomains([&](auto& loc_dom) { - Kokkos::parallel_for( - "InitFields", - loc_dom.mesh.rangeActiveCells(), - arch::SetEMFields_kernel { - loc_dom.fields.em, - m_pgen.init_flds, - loc_dom.mesh.metric }); - }); - } - if constexpr ( - traits::has_member>::value) { - logger::Checkpoint("Initializing particles from problem generator", HERE); - m_metadomain.runOnLocalDomains([&](auto& loc_dom) { - m_pgen.InitPrtls(loc_dom); - }); - } - } else { -#if defined(OUTPUT_ENABLED) - // read simulation data from the checkpoint - raise::ErrorIf( - m_params.template get("checkpoint.start_step") == 0, - "Resuming simulation from a checkpoint requires a valid start_step", - HERE); - logger::Checkpoint("Resuming simulation from a checkpoint", HERE); - m_metadomain.ContinueFromCheckpoint(&m_adios, m_params); -#else - raise::Error( - "Resuming simulation from a checkpoint requires -D output=ON", - HERE); -#endif - } - } - print_report(); - } - -#define ENGINE_INIT(S, M, D) template class Engine>; - - NTT_FOREACH_SPECIALIZATION(ENGINE_INIT) - -#undef ENGINE_INIT - -} // namespace ntt diff --git a/src/engines/engine_printer.cpp b/src/engines/engine_printer.cpp deleted file mode 100644 index f3d6f3b38..000000000 --- a/src/engines/engine_printer.cpp +++ /dev/null @@ -1,500 +0,0 @@ -#include "enums.h" -#include "global.h" - -#include "arch/directions.h" -#include "arch/mpi_aliases.h" -#include "utils/colors.h" -#include "utils/formatting.h" - -#include "framework/specialization_registry.h" - -#include "engines/engine.hpp" - -#if defined(CUDA_ENABLED) - #include -#elif defined(HIP_ENABLED) - #include -#endif - -#if defined(OUTPUT_ENABLED) - #include -#endif - -#include -#include -#include - -namespace ntt { - - namespace { - void add_header(std::string& report, - const std::vector& lines, - const std::vector& colors) { - report += fmt::format("%s╔%s╗%s\n", - color::BRIGHT_BLACK, - fmt::repeat("═", 58).c_str(), - color::RESET); - for (auto i { 0u }; i < lines.size(); ++i) { - report += fmt::format("%s║%s %s%s%s%s%s║%s\n", - color::BRIGHT_BLACK, - color::RESET, - colors[i], - lines[i].c_str(), - color::RESET, - fmt::repeat(" ", 57 - lines[i].size()).c_str(), - color::BRIGHT_BLACK, - color::RESET); - } - report += fmt::format("%s╚%s╝%s\n", - color::BRIGHT_BLACK, - fmt::repeat("═", 58).c_str(), - color::RESET); - } - - void add_category(std::string& report, unsigned short indent, const char* name) { - report += fmt::format("%s%s%s%s\n", - std::string(indent, ' ').c_str(), - color::BLUE, - name, - color::RESET); - } - - void add_subcategory(std::string& report, unsigned short indent, const char* name) { - report += fmt::format("%s%s-%s %s:\n", - std::string(indent, ' ').c_str(), - color::BRIGHT_BLACK, - color::RESET, - name); - } - - void add_label(std::string& report, unsigned short indent, const char* label) { - report += fmt::format("%s%s\n", std::string(indent, ' ').c_str(), label); - } - - template - void add_param(std::string& report, - unsigned short indent, - const char* name, - const char* format, - Args... args) { - report += fmt::format("%s%s-%s %s: %s%s%s\n", - std::string(indent, ' ').c_str(), - color::BRIGHT_BLACK, - color::RESET, - name, - color::BRIGHT_YELLOW, - fmt::format(format, args...).c_str(), - color::RESET); - } - - template - void add_unlabeled_param(std::string& report, - unsigned short indent, - const char* name, - const char* format, - Args... args) { - report += fmt::format("%s%s: %s%s%s\n", - std::string(indent, ' ').c_str(), - name, - color::BRIGHT_YELLOW, - fmt::format(format, args...).c_str(), - color::RESET); - } - - auto bytes_to_human_readable( - std::size_t bytes) -> std::pair { - const std::vector units { "B", "KB", "MB", "GB", "TB" }; - idx_t unit_idx = 0; - auto size = static_cast(bytes); - while ((size >= 1024.0) and (unit_idx < units.size() - 1)) { - size /= 1024.0; - ++unit_idx; - } - return { size, units[unit_idx] }; - } - } // namespace - - template - void Engine::print_report() const { - const auto colored_stdout = m_params.template get( - "diagnostics.colored_stdout"); - std::string report = ""; - CallOnce( - [&](auto& metadomain, auto& params) { -#if defined(MPI_ENABLED) - int mpi_v, mpi_subv; - MPI_Get_version(&mpi_v, &mpi_subv); - const std::string mpi_version = fmt::format("%d.%d", mpi_v, mpi_subv); -#else // not MPI_ENABLED - const std::string mpi_version = "OFF"; -#endif // MPI_ENABLED - - const auto entity_version = "Entity v" + std::string(ENTITY_VERSION); - const auto hash = std::string(ENTITY_GIT_HASH); - const auto pgen = std::string(PGEN); - const auto nspec = metadomain.species_params().size(); - const auto precision = (sizeof(real_t) == 4) ? "single" : "double"; - -#if defined(__clang__) - const std::string ccx = "Clang/LLVM " __clang_version__; -#elif defined(__ICC) || defined(__INTEL_COMPILER) - const std::string ccx = "Intel ICC/ICPC " __VERSION__; -#elif defined(__GNUC__) || defined(__GNUG__) - const std::string ccx = "GNU GCC/G++ " __VERSION__; -#elif defined(__HP_cc) || defined(__HP_aCC) - const std::string ccx = "Hewlett-Packard C/aC++ " __HP_aCC; -#elif defined(__IBMC__) || defined(__IBMCPP__) - const std::string ccx = "IBM XL C/C++ " __IBMCPP__; -#elif defined(_MSC_VER) - const std::string ccx = "Microsoft Visual Studio " _MSC_VER; -#else - const std::string ccx = "Unknown compiler"; -#endif - std::string cpp_standard; - if (__cplusplus == 202101L) { - cpp_standard = "C++23"; - } else if (__cplusplus == 202002L) { - cpp_standard = "C++20"; - } else if (__cplusplus == 201703L) { - cpp_standard = "C++17"; - } else if (__cplusplus == 201402L) { - cpp_standard = "C++14"; - } else if (__cplusplus == 201103L) { - cpp_standard = "C++11"; - } else if (__cplusplus == 199711L) { - cpp_standard = "C++98"; - } else { - cpp_standard = "pre-standard " + std::to_string(__cplusplus); - } - -#if defined(CUDA_ENABLED) - int cuda_v; - cudaRuntimeGetVersion(&cuda_v); - const auto major { cuda_v / 1000 }; - const auto minor { cuda_v % 1000 / 10 }; - const auto patch { cuda_v % 10 }; - const auto cuda_version = fmt::format("%d.%d.%d", major, minor, patch); -#elif defined(HIP_ENABLED) - int hip_v; - auto status = hipDriverGetVersion(&hip_v); - raise::ErrorIf(status != hipSuccess, - "hipDriverGetVersion failed with error code %d", - HERE); - const auto major { hip_v / 10000000 }; - const auto minor { (hip_v % 10000000) / 100000 }; - const auto patch { hip_v % 100000 }; - const auto hip_version = fmt::format("%d.%d.%d", major, minor, patch); -#endif - - const auto kokkos_version = fmt::format("%d.%d.%d", - KOKKOS_VERSION / 10000, - KOKKOS_VERSION / 100 % 100, - KOKKOS_VERSION % 100); - -#if defined(OUTPUT_ENABLED) - const std::string adios2_version = fmt::format("%d.%d.%d", - ADIOS2_VERSION / 10000, - ADIOS2_VERSION / 100 % 100, - ADIOS2_VERSION % 100); -#else // not OUTPUT_ENABLED - const std::string adios2_version = "OFF"; -#endif - -#if defined(DEBUG) - const std::string dbg = "ON"; -#else // not DEBUG - const std::string dbg = "OFF"; -#endif - - report += "\n\n"; - add_header(report, { entity_version }, { color::BRIGHT_GREEN }); - report += "\n"; - - /* - * Backend - */ - add_category(report, 4, "Backend"); - add_param(report, 4, "Build hash", "%s", hash.c_str()); - add_param(report, 4, "CXX", "%s [%s]", ccx.c_str(), cpp_standard.c_str()); -#if defined(CUDA_ENABLED) - add_param(report, 4, "CUDA", "%s", cuda_version.c_str()); -#elif defined(HIP_VERSION) - add_param(report, 4, "HIP", "%s", hip_version.c_str()); -#endif - add_param(report, 4, "MPI", "%s", mpi_version.c_str()); -#if defined(MPI_ENABLED) && defined(DEVICE_ENABLED) - #if defined(GPU_AWARE_MPI) - const std::string gpu_aware_mpi = "ON"; - #else - const std::string gpu_aware_mpi = "OFF"; - #endif - add_param(report, 4, "GPU-aware MPI", "%s", gpu_aware_mpi.c_str()); -#endif - add_param(report, 4, "Kokkos", "%s", kokkos_version.c_str()); - add_param(report, 4, "ADIOS2", "%s", adios2_version.c_str()); - add_param(report, 4, "Precision", "%s", precision); - add_param(report, 4, "Debug", "%s", dbg.c_str()); - report += "\n"; - - /* - * Compilation flags - */ - add_category(report, 4, "Compilation flags"); -#if defined(SINGLE_PRECISION) - add_param(report, 4, "SINGLE_PRECISION", "%s", "ON"); -#else - add_param(report, 4, "SINGLE_PRECISION", "%s", "OFF"); -#endif - -#if defined(OUTPUT_ENABLED) - add_param(report, 4, "OUTPUT_ENABLED", "%s", "ON"); -#else - add_param(report, 4, "OUTPUT_ENABLED", "%s", "OFF"); -#endif - -#if defined(DEBUG) - add_param(report, 4, "DEBUG", "%s", "ON"); -#else - add_param(report, 4, "DEBUG", "%s", "OFF"); -#endif - -#if defined(CUDA_ENABLED) - add_param(report, 4, "CUDA_ENABLED", "%s", "ON"); -#else - add_param(report, 4, "CUDA_ENABLED", "%s", "OFF"); -#endif - -#if defined(HIP_ENABLED) - add_param(report, 4, "HIP_ENABLED", "%s", "ON"); -#else - add_param(report, 4, "HIP_ENABLED", "%s", "OFF"); -#endif - -#if defined(DEVICE_ENABLED) - add_param(report, 4, "DEVICE_ENABLED", "%s", "ON"); -#else - add_param(report, 4, "DEVICE_ENABLED", "%s", "OFF"); -#endif - -#if defined(MPI_ENABLED) - add_param(report, 4, "MPI_ENABLED", "%s", "ON"); -#else - add_param(report, 4, "MPI_ENABLED", "%s", "OFF"); -#endif - -#if defined(GPU_AWARE_MPI) - add_param(report, 4, "GPU_AWARE_MPI", "%s", "ON"); -#else - add_param(report, 4, "GPU_AWARE_MPI", "%s", "OFF"); -#endif - report += "\n"; - - /* - * Simulation configs - */ - add_category(report, 4, "Configuration"); - add_param(report, - 4, - "Name", - "%s", - params.template get("simulation.name").c_str()); - add_param(report, 4, "Problem generator", "%s", pgen.c_str()); - add_param(report, 4, "Engine", "%s", SimEngine(S).to_string()); - add_param(report, 4, "Metric", "%s", Metric(M::MetricType).to_string()); -#if SHAPE_ORDER == 0 - add_param(report, 4, "Deposit", "%s", "zigzag"); -#else - add_param(report, 4, "Deposit", "%s", "esirkepov"); - add_param(report, 4, "Interpolation order", "%i", SHAPE_ORDER); -#endif - add_param(report, 4, "Timestep [dt]", "%.3e", dt); - add_param(report, 4, "Runtime", "%.3e [%d steps]", runtime, max_steps); - report += "\n"; - add_category(report, 4, "Global domain"); - add_param(report, - 4, - "Resolution", - "%s", - params.template stringize("grid.resolution").c_str()); - add_param(report, - 4, - "Extent", - "%s", - params.template stringize("grid.extent").c_str()); - add_param(report, - 4, - "Fiducial cell size [dx0]", - "%.3e", - params.template get("scales.dx0")); - add_subcategory(report, 4, "Boundary conditions"); - add_param( - report, - 6, - "Fields", - "%s", - params.template stringize("grid.boundaries.fields").c_str()); - add_param( - report, - 6, - "Particles", - "%s", - params.template stringize("grid.boundaries.particles").c_str()); - add_param(report, - 4, - "Domain decomposition", - "%s [%d total]", - fmt::formatVector(m_metadomain.ndomains_per_dim()).c_str(), - m_metadomain.ndomains()); - report += "\n"; - add_category(report, 4, "Fiducial parameters"); - add_param(report, - 4, - "Particles per cell [ppc0]", - "%.1f", - params.template get("particles.ppc0")); - add_param(report, - 4, - "Larmor radius [larmor0]", - "%.3e [%.3f dx0]", - params.template get("scales.larmor0"), - params.template get("scales.larmor0") / - params.template get("scales.dx0")); - add_param(report, - 4, - "Larmor frequency [omegaB0 * dt]", - "%.3e", - params.template get("scales.omegaB0") * - params.template get("algorithms.timestep.dt")); - add_param(report, - 4, - "Skin depth [skindepth0]", - "%.3e [%.3f dx0]", - params.template get("scales.skindepth0"), - params.template get("scales.skindepth0") / - params.template get("scales.dx0")); - add_param(report, - 4, - "Plasma frequency [omp0 * dt]", - "%.3e", - params.template get("algorithms.timestep.dt") / - params.template get("scales.skindepth0")); - add_param(report, - 4, - "Magnetization [sigma0]", - "%.3e", - params.template get("scales.sigma0")); - - if (nspec > 0) { - report += "\n"; - add_category(report, 4, "Particles"); - } - for (const auto& species : metadomain.species_params()) { - add_subcategory(report, - 4, - fmt::format("Species #%d", species.index()).c_str()); - add_param(report, 6, "Label", "%s", species.label().c_str()); - add_param(report, 6, "Mass", "%.1f", species.mass()); - add_param(report, 6, "Charge", "%.1f", species.charge()); - add_param(report, 6, "Max #", "%d [per domain]", species.maxnpart()); - add_param(report, 6, "Pusher", "%s", species.pusher().to_string()); - if (species.mass() != 0.0) { - add_param(report, 6, "GCA", "%s", species.use_gca() ? "ON" : "OFF"); - } - add_param(report, 6, "Cooling", "%s", species.cooling().to_string()); - add_param(report, 6, "# of real-value payloads", "%d", species.npld_r()); - add_param(report, 6, "# of integer-value payloads", "%d", species.npld_i()); - } - report.pop_back(); - }, - m_metadomain, - m_params); - info::Print(report, colored_stdout); - - report = "\n"; - CallOnce([&]() { - add_category(report, 4, "Domains"); - report.pop_back(); - }); - info::Print(report, colored_stdout); - - for (unsigned int idx { 0 }; idx < m_metadomain.ndomains(); ++idx) { - auto is_local = false; - for (const auto& lidx : m_metadomain.l_subdomain_indices()) { - is_local |= (idx == lidx); - } - if (is_local) { - report = ""; - const auto& domain = m_metadomain.subdomain(idx); - add_subcategory(report, - 4, - fmt::format("Domain #%d", domain.index()).c_str()); -#if defined(MPI_ENABLED) - add_param(report, 6, "Rank", "%d", domain.mpi_rank()); -#endif - add_param(report, - 6, - "Resolution", - "%s", - fmt::formatVector(domain.mesh.n_active()).c_str()); - add_param(report, - 6, - "Extent", - "%s", - fmt::formatVector(domain.mesh.extent()).c_str()); - add_subcategory(report, 6, "Boundary conditions"); - - add_label( - report, - 8 + 2 + 2 * M::Dim, - fmt::format("%-10s %-10s %-10s", "[flds]", "[prtl]", "[neighbor]").c_str()); - for (auto& direction : dir::Directions::all) { - const auto flds_bc = domain.mesh.flds_bc_in(direction); - const auto prtl_bc = domain.mesh.prtl_bc_in(direction); - bool has_sync = false; - auto neighbor_idx = domain.neighbor_idx_in(direction); - if (flds_bc == FldsBC::SYNC || prtl_bc == PrtlBC::SYNC) { - has_sync = true; - } - add_unlabeled_param(report, - 8, - direction.to_string().c_str(), - "%-10s %-10s %-10s", - flds_bc.to_string(), - prtl_bc.to_string(), - has_sync ? std::to_string(neighbor_idx).c_str() - : "."); - } - add_subcategory(report, 6, "Memory footprint"); - auto flds_footprint = domain.fields.memory_footprint(); - auto [flds_size, flds_unit] = bytes_to_human_readable(flds_footprint); - add_param(report, 8, "Fields", "%.2f %s", flds_size, flds_unit.c_str()); - if (domain.species.size() > 0) { - add_subcategory(report, 8, "Particles"); - } - for (auto& species : domain.species) { - const auto str = fmt::format("Species #%d (%s)", - species.index(), - species.label().c_str()); - auto [size, unit] = bytes_to_human_readable(species.memory_footprint()); - add_param(report, 10, str.c_str(), "%.2f %s", size, unit.c_str()); - } - report.pop_back(); - if (idx == m_metadomain.ndomains() - 1) { - report += "\n\n"; - } - info::Print(report, colored_stdout, true, false); - } -#if defined(MPI_ENABLED) - MPI_Barrier(MPI_COMM_WORLD); -#endif - } - } - -#define ENGINE_PRINTER(S, M, D) \ - template void Engine>::print_report() const; - - NTT_FOREACH_SPECIALIZATION(ENGINE_PRINTER) - -#undef ENGINE_PRINTER - -} // namespace ntt diff --git a/src/engines/engine_run.cpp b/src/engines/engine_run.cpp deleted file mode 100644 index 961e39206..000000000 --- a/src/engines/engine_run.cpp +++ /dev/null @@ -1,146 +0,0 @@ -#include "enums.h" - -#include "arch/traits.h" -#include "utils/diag.h" - -#include "framework/domain/domain.h" -#include "framework/specialization_registry.h" - -#include "engines/engine.hpp" - -namespace ntt { - - template - void Engine::run() { - if constexpr (pgen_is_ok) { - init(); - - auto timers = timer::Timers { - { "FieldSolver", - "CurrentFiltering", "CurrentDeposit", - "ParticlePusher", "FieldBoundaries", - "ParticleBoundaries", "Communications", - "Injector", "Custom", - "PrtlClear", "Output", - "Checkpoint" }, - []() { - Kokkos::fence(); - }, - m_params.get("diagnostics.blocking_timers") - }; - const auto diag_interval = m_params.get( - "diagnostics.interval"); - - auto time_history = pbar::DurationHistory { 1000 }; - const auto clear_interval = m_params.template get( - "particles.clear_interval"); - - // main algorithm loop - while (step < max_steps) { - // run the engine-dependent algorithm step - m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - step_forward(timers, dom); - }); - // poststep (if defined) - if constexpr ( - traits::has_method::value) { - timers.start("Custom"); - m_metadomain.runOnLocalDomains([&timers, this](auto& dom) { - m_pgen.CustomPostStep(step, time, dom); - }); - timers.stop("Custom"); - } - auto print_prtl_clear = (clear_interval > 0 and - step % clear_interval == 0 and step > 0); - - // advance time & step - time += dt; - ++step; - - auto print_output = false; - auto print_checkpoint = false; -#if defined(OUTPUT_ENABLED) - timers.start("Output"); - if constexpr ( - traits::has_method::value) { - auto lambda_custom_field_output = [&](const std::string& name, - ndfield_t& buff, - index_t idx, - timestep_t step, - simtime_t time, - const Domain& dom) { - m_pgen.CustomFieldOutput(name, buff, idx, step, time, dom); - }; - print_output &= m_metadomain.Write(m_params, - step, - step - 1, - time, - time - dt, - lambda_custom_field_output); - } else { - print_output &= m_metadomain.Write(m_params, step, step - 1, time, time - dt); - } - if constexpr ( - traits::has_method::value) { - auto lambda_custom_stat = [&](const std::string& name, - timestep_t step, - simtime_t time, - const Domain& dom) -> real_t { - return m_pgen.CustomStat(name, step, time, dom); - }; - print_output &= m_metadomain.WriteStats(m_params, - step, - step - 1, - time, - time - dt, - lambda_custom_stat); - } else { - print_output &= m_metadomain.WriteStats(m_params, - step, - step - 1, - time, - time - dt); - } - timers.stop("Output"); - - timers.start("Checkpoint"); - print_checkpoint = m_metadomain.WriteCheckpoint(m_params, - step, - step - 1, - time, - time - dt); - timers.stop("Checkpoint"); -#endif - - // advance time_history - time_history.tick(); - // print timestep report - if (diag_interval > 0 and step % diag_interval == 0) { - diag::printDiagnostics( - step - 1, - max_steps, - time - dt, - dt, - timers, - time_history, - m_metadomain.l_ncells(), - m_metadomain.species_labels(), - m_metadomain.l_npart_perspec(), - m_metadomain.l_maxnpart_perspec(), - print_prtl_clear, - print_output, - print_checkpoint, - m_params.get("diagnostics.colored_stdout")); - } - timers.resetAll(); - } - } - } - -#define ENGINE_RUN(S, M, D) template void Engine>::run(); - - NTT_FOREACH_SPECIALIZATION(ENGINE_RUN) - -#undef ENGINE_RUN - -} // namespace ntt diff --git a/src/engines/engine_traits.h b/src/engines/engine_traits.h deleted file mode 100644 index 28a74bea3..000000000 --- a/src/engines/engine_traits.h +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#ifndef ENGINES_ENGINE_TRAITS_H -#define ENGINES_ENGINE_TRAITS_H - -#include "enums.h" - -#include "engines/grpic.hpp" -#include "engines/srpic.hpp" - -namespace ntt { - - template - struct EngineSelector; - - template <> - struct EngineSelector { - template - using type = SRPICEngine; - }; - - template <> - struct EngineSelector { - template - using type = GRPICEngine; - }; - -} // namespace ntt - -#endif // ENGINES_ENGINE_TRAITS_H diff --git a/src/engines/grpic.hpp b/src/engines/grpic.hpp index 4d9d89d92..7952153ba 100644 --- a/src/engines/grpic.hpp +++ b/src/engines/grpic.hpp @@ -19,12 +19,9 @@ #include "utils/log.h" #include "utils/numeric.h" #include "utils/timer.h" -#include "utils/toml.h" #include "framework/domain/domain.h" -#include "framework/parameters.h" - -#include "engines/engine.hpp" +#include "framework/parameters/parameters.h" #include "kernels/ampere_gr.hpp" #include "kernels/aux_fields_gr.hpp" #include "kernels/currents_deposit.hpp" @@ -32,10 +29,13 @@ #include "kernels/faraday_gr.hpp" #include "kernels/fields_bcs.hpp" #include "kernels/particle_pusher_gr.hpp" + +#include "engines/engine.hpp" #include "pgen.hpp" #include #include +#include #include #include @@ -66,12 +66,11 @@ namespace ntt { }; template + requires traits::engine::IsCompatibleWithGRPICEngine class GRPICEngine : public Engine { using base_t = Engine; using pgen_t = user::PGen; using domain_t = Domain; - // constexprs - using base_t::pgen_is_ok; // contents using base_t::m_metadomain; using base_t::m_params; @@ -533,11 +532,9 @@ namespace ntt { timers.stop("FieldBoundaries"); } - if (clear_interval > 0 and step % clear_interval == 0 and step > 0) { - timers.start("PrtlClear"); - m_metadomain.RemoveDeadParticles(dom); - timers.stop("PrtlClear"); - } + timers.start("ParticleSort"); + m_metadomain.SortParticles(time, step, m_params, dom); + timers.stop("ParticleSort"); /** * Finally: em0::B at n-1/2 @@ -638,28 +635,30 @@ namespace ntt { } if (dim == in::x1) { if (g != gr_bc::curr) { - Kokkos::parallel_for( - "MatchBoundaries", - CreateRangePolicy(range_min, range_max), - kernel::bc::MatchBoundaries_kernel( - domain.fields.em, - m_pgen.init_flds, - domain.mesh.metric, - xg_edge, - ds, - tags, - domain.mesh.flds_bc())); - Kokkos::parallel_for( - "MatchBoundaries", - CreateRangePolicy(range_min, range_max), - kernel::bc::MatchBoundaries_kernel( - domain.fields.em0, - m_pgen.init_flds, - domain.mesh.metric, - xg_edge, - ds, - tags, - domain.mesh.flds_bc())); + if constexpr (arch::traits::pgen::HasInitFlds) { + Kokkos::parallel_for( + "MatchBoundaries", + CreateRangePolicy(range_min, range_max), + kernel::bc::MatchBoundaries_kernel( + domain.fields.em, + m_pgen.init_flds, + domain.mesh.metric, + xg_edge, + ds, + tags, + domain.mesh.flds_bc())); + Kokkos::parallel_for( + "MatchBoundaries", + CreateRangePolicy(range_min, range_max), + kernel::bc::MatchBoundaries_kernel( + domain.fields.em0, + m_pgen.init_flds, + domain.mesh.metric, + xg_edge, + ds, + tags, + domain.mesh.flds_bc())); + } } else { Kokkos::parallel_for( "AbsorbCurrents", @@ -696,17 +695,17 @@ namespace ntt { Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::gr::HorizonBoundaries_kernel(domain.fields.em, - i1_min, - tags, - nfilter)); + kernel::bc::gr::HorizonBoundaries_kernel(domain.fields.em, + i1_min, + tags, + nfilter)); Kokkos::parallel_for( "OpenBCFields", range, - kernel::bc::gr::HorizonBoundaries_kernel(domain.fields.em0, - i1_min, - tags, - nfilter)); + kernel::bc::gr::HorizonBoundaries_kernel(domain.fields.em0, + i1_min, + tags, + nfilter)); } } @@ -974,19 +973,19 @@ namespace ntt { } void TimeAverageDB(domain_t& domain) { - Kokkos::parallel_for("TimeAverageDB", - domain.mesh.rangeActiveCells(), - kernel::gr::TimeAverageDB_kernel(domain.fields.em, - domain.fields.em0, - domain.mesh.metric)); + Kokkos::parallel_for( + "TimeAverageDB", + domain.mesh.rangeActiveCells(), + kernel::gr::TimeAverageDB_kernel(domain.fields.em, + domain.fields.em0)); } void TimeAverageJ(domain_t& domain) { - Kokkos::parallel_for("TimeAverageJ", - domain.mesh.rangeActiveCells(), - kernel::gr::TimeAverageJ_kernel(domain.fields.cur, - domain.fields.cur0, - domain.mesh.metric)); + Kokkos::parallel_for( + "TimeAverageJ", + domain.mesh.rangeActiveCells(), + kernel::gr::TimeAverageJ_kernel(domain.fields.cur, + domain.fields.cur0)); } void CurrentsDeposit(domain_t& domain) { @@ -1081,36 +1080,40 @@ namespace ntt { "algorithms.gr.pusher_eps"); const auto niter = m_params.template get( "algorithms.gr.pusher_niter"); - // clang-format off - if (species.pusher() == PrtlPusher::PHOTON) { - auto range_policy = Kokkos::RangePolicy( - 0, - species.npart()); + if (species.pusher() == ParticlePusher::PHOTON) { + auto range_policy = + Kokkos::RangePolicy( + 0, + species.npart()); - Kokkos::parallel_for( - "ParticlePusher", - range_policy, - kernel::gr::Pusher_kernel( - domain.fields.em, - domain.fields.em0, - species.i1, species.i2, species.i3, - species.i1_prev, species.i2_prev, species.i3_prev, - species.dx1, species.dx2, species.dx3, - species.dx1_prev, species.dx2_prev, species.dx3_prev, - species.ux1, species.ux2, species.ux3, - species.phi, species.tag, - domain.mesh.metric, - coeff, dt, - domain.mesh.n_active(in::x1), - domain.mesh.n_active(in::x2), - domain.mesh.n_active(in::x3), - eps, niter, - domain.mesh.prtl_bc() - )); - } else if (species.pusher() == PrtlPusher::BORIS) { - auto range_policy = Kokkos::RangePolicy( - 0, - species.npart()); + // clang-format off + Kokkos::parallel_for( + "ParticlePusher", + range_policy, + kernel::gr::Pusher_kernel( + domain.fields.em, + domain.fields.em0, + species.i1, species.i2, species.i3, + species.i1_prev, species.i2_prev, species.i3_prev, + species.dx1, species.dx2, species.dx3, + species.dx1_prev, species.dx2_prev, species.dx3_prev, + species.ux1, species.ux2, species.ux3, + species.phi, species.tag, + domain.mesh.metric, + coeff, dt, + domain.mesh.n_active(in::x1), + domain.mesh.n_active(in::x2), + domain.mesh.n_active(in::x3), + eps, niter, + domain.mesh.prtl_bc() + )); + // clang-format on + } else if (species.pusher() == ParticlePusher::BORIS) { + auto range_policy = + Kokkos::RangePolicy( + 0, + species.npart()); + // clang-format off Kokkos::parallel_for( "ParticlePusher", range_policy, @@ -1131,12 +1134,12 @@ namespace ntt { eps, niter, domain.mesh.prtl_bc() )); - } else if (species.pusher() == PrtlPusher::NONE) { + // clang-format on + } else if (species.pusher() == ParticlePusher::NONE) { // do nothing } else { raise::Error("not implemented", HERE); } - // clang-format on } } }; diff --git a/src/engines/reporter.cpp b/src/engines/reporter.cpp new file mode 100644 index 000000000..191e5b732 --- /dev/null +++ b/src/engines/reporter.cpp @@ -0,0 +1,153 @@ +#include "engines/reporter.h" + +#include "enums.h" + +#include "utils/reporter.h" + +#include "framework/parameters/parameters.h" + +#include +#include + +namespace ntt { + + auto ReportSimulationConfig(const SimulationParams& params, + SimEngine S, + Metric M, + real_t dt, + simtime_t runtime, + timestep_t max_steps, + const std::vector& ndomains_per_dim, + unsigned int ndomains) -> std::string { + std::string report = ""; + /* + * Simulation configs + */ + reporter::AddCategory(report, 4, "Configuration"); + reporter::AddParam(report, + 4, + "Name", + "%s", + params.template get("simulation.name").c_str()); + reporter::AddParam(report, 4, "Engine", "%s", S.to_string()); + reporter::AddParam(report, 4, "Metric", "%s", M.to_string()); +#if SHAPE_ORDER == 0 + reporter::AddParam(report, 4, "Deposit", "%s", "zigzag"); +#else + reporter::AddParam(report, 4, "Deposit", "%s", "esirkepov"); + reporter::AddParam(report, 4, "Interpolation order", "%i", SHAPE_ORDER); +#endif + reporter::AddParam(report, 4, "Timestep [dt]", "%.3e", dt); + reporter::AddParam(report, 4, "Runtime", "%.3e [%d steps]", runtime, max_steps); + report += "\n"; + reporter::AddCategory(report, 4, "Global domain"); + reporter::AddParam( + report, + 4, + "Resolution", + "%s", + params.template stringize("grid.resolution").c_str()); + reporter::AddParam(report, + 4, + "Extent", + "%s", + params.template stringize("grid.extent").c_str()); + reporter::AddParam(report, + 4, + "Fiducial cell size [dx0]", + "%.3e", + params.template get("scales.dx0")); + reporter::AddSubcategory(report, 4, "Boundary conditions"); + reporter::AddParam( + report, + 6, + "Fields", + "%s", + params.template stringize("grid.boundaries.fields").c_str()); + reporter::AddParam( + report, + 6, + "Particles", + "%s", + params.template stringize("grid.boundaries.particles").c_str()); + reporter::AddParam(report, + 4, + "Domain decomposition", + "%s [%d total]", + fmt::formatVector(ndomains_per_dim).c_str(), + ndomains); + report += "\n"; + reporter::AddCategory(report, 4, "Nominal parameters"); + reporter::AddParam(report, + 4, + "Particles per cell [ppc0]", + "%.1f", + params.template get("particles.ppc0")); + reporter::AddParam(report, + 4, + "Larmor radius [larmor0]", + "%.3e [%.3f dx0]", + params.template get("scales.larmor0"), + params.template get("scales.larmor0") / + params.template get("scales.dx0")); + reporter::AddParam(report, + 4, + "Larmor frequency [omegaB0 * dt]", + "%.3e", + params.template get("scales.omegaB0") * + params.template get("algorithms.timestep.dt")); + reporter::AddParam(report, + 4, + "Skin depth [skindepth0]", + "%.3e [%.3f dx0]", + params.template get("scales.skindepth0"), + params.template get("scales.skindepth0") / + params.template get("scales.dx0")); + reporter::AddParam(report, + 4, + "Plasma frequency [omp0 * dt]", + "%.3e", + params.template get("algorithms.timestep.dt") / + params.template get("scales.skindepth0")); + reporter::AddParam(report, + 4, + "Magnetization [sigma0]", + "%.3e", + params.template get("scales.sigma0")); + + if (params.contains("radiation.emission.compton.photon_species")) { + reporter::AddCategory(report, 4, "- Compton emission"); + reporter::AddParam(report, + 6, + "Nominal probability", + "%.3e", + params.template get( + "radiation.emission.compton.nominal_probability")); + reporter::AddParam(report, + 6, + "Nominal photon energy", + "%.3e", + params.template get( + "radiation.emission.compton.nominal_photon_energy")); + } + if (params.contains("radiation.emission.synchrotron.photon_species")) { + reporter::AddCategory(report, 4, "- Synchrotron emission"); + reporter::AddParam( + report, + 6, + "Nominal probability", + "%.3e", + params.template get( + "radiation.emission.synchrotron.nominal_probability")); + reporter::AddParam( + report, + 6, + "Nominal photon energy", + "%.3e", + params.template get( + "radiation.emission.synchrotron.nominal_photon_energy")); + } + return report; + } + +} // namespace ntt diff --git a/src/engines/reporter.h b/src/engines/reporter.h new file mode 100644 index 000000000..9cbcf3a3d --- /dev/null +++ b/src/engines/reporter.h @@ -0,0 +1,115 @@ +#ifndef ENGINES_REPORTER_H +#define ENGINES_REPORTER_H + +#include "enums.h" + +#include "utils/reporter.h" + +#include "archetypes/traits.h" +#include "framework/parameters/parameters.h" + +#include +#include + +namespace ntt { + + auto ReportSimulationConfig(const SimulationParams&, + SimEngine, + Metric, + real_t, + simtime_t, + timestep_t, + const std::vector&, + unsigned int) -> std::string; + + template + inline auto ReportPgenConfig(const PG& pgen, const std::string& pgen_name) + -> std::string { + std::string report = ""; + report += "\n"; + reporter::AddCategory(report, 4, "Problem generator"); + reporter::AddParam(report, 6, "Name", "%s", pgen_name.c_str()); + reporter::AddSubcategory(report, 6, "Methods defined"); + + const auto BoolToOnOff = [](bool toggle) -> const char* { + return toggle ? "ON" : "OFF"; + }; + + reporter::AddParam(report, + 8, + "InitFlds", + "%s", + BoolToOnOff(arch::traits::pgen::HasInitFlds)); + reporter::AddParam(report, + 8, + "InitPrtls", + "%s", + BoolToOnOff(arch::traits::pgen::HasInitPrtls)); + reporter::AddParam(report, + 8, + "CustomPostStep", + "%s", + BoolToOnOff(arch::traits::pgen::HasCustomPostStep)); + reporter::AddParam(report, + 8, + "ExternalFields", + "%s", + BoolToOnOff(arch::traits::pgen::HasExternalFields)); + reporter::AddParam(report, + 8, + "ext_current", + "%s", + BoolToOnOff(arch::traits::pgen::HasExtCurrent)); + reporter::AddParam(report, + 8, + "AtmFields", + "%s", + BoolToOnOff(arch::traits::pgen::HasAtmFields)); + reporter::AddParam(report, + 8, + "MatchFields", + "%s", + BoolToOnOff(arch::traits::pgen::HasMatchFields)); + reporter::AddParam(report, + 8, + "MatchFieldsInX1", + "%s", + BoolToOnOff(arch::traits::pgen::HasMatchFieldsInX1)); + reporter::AddParam(report, + 8, + "MatchFieldsInX2", + "%s", + BoolToOnOff(arch::traits::pgen::HasMatchFieldsInX2)); + reporter::AddParam(report, + 8, + "MatchFieldsInX3", + "%s", + BoolToOnOff(arch::traits::pgen::HasMatchFieldsInX3)); + reporter::AddParam(report, + 8, + "FixFieldsConst", + "%s", + BoolToOnOff(arch::traits::pgen::HasFixFieldsConst)); + reporter::AddParam(report, + 8, + "EmissionPolicy", + "%s", + BoolToOnOff(arch::traits::pgen::HasEmissionPolicy)); + reporter::AddParam( + report, + 8, + "CustomFieldOutput", + "%s", + BoolToOnOff(arch::traits::pgen::HasCustomFieldOutput)); + reporter::AddParam( + report, + 8, + "CustomStatOutput", + "%s", + BoolToOnOff(arch::traits::pgen::HasCustomStatOutput)); + return report; + } + +} // namespace ntt + +#endif // ENGINES_REPORTER_H diff --git a/src/engines/srpic.hpp b/src/engines/srpic.hpp deleted file mode 100644 index 0dad75e91..000000000 --- a/src/engines/srpic.hpp +++ /dev/null @@ -1,1569 +0,0 @@ -/** - * @file engines/srpic.hpp - * @brief Simulation engien class which specialized on SRPIC - * @implements - * - ntt::SRPICEngine<> : ntt::Engine<> - * @cpp: - * - srpic.cpp - * @namespaces: - * - ntt:: - * @macros: - */ - -#ifndef ENGINES_SRPIC_SRPIC_H -#define ENGINES_SRPIC_SRPIC_H - -#include "enums.h" -#include "global.h" - -#include "arch/kokkos_aliases.h" -#include "arch/traits.h" -#include "utils/log.h" -#include "utils/numeric.h" -#include "utils/timer.h" -#include "utils/toml.h" - -#include "archetypes/energy_dist.h" -#include "archetypes/particle_injector.h" -#include "archetypes/spatial_dist.h" -#include "framework/domain/domain.h" -#include "framework/parameters.h" - -#include "engines/engine.hpp" -#include "kernels/ampere_mink.hpp" -#include "kernels/ampere_sr.hpp" -#include "kernels/currents_deposit.hpp" -#include "kernels/digital_filter.hpp" -#include "kernels/faraday_mink.hpp" -#include "kernels/faraday_sr.hpp" -#include "kernels/fields_bcs.hpp" -#include "kernels/particle_moments.hpp" -#include "kernels/particle_pusher_sr.hpp" -#include "pgen.hpp" - -#include -#include - -#include - -namespace ntt { - - template - class SRPICEngine : public Engine { - - using base_t = Engine; - using pgen_t = user::PGen; - using domain_t = Domain; - // constexprs - using base_t::pgen_is_ok; - // contents - using base_t::m_metadomain; - using base_t::m_params; - using base_t::m_pgen; - // methods - using base_t::init; - // variables - using base_t::dt; - using base_t::max_steps; - using base_t::runtime; - using base_t::step; - using base_t::time; - - public: - static constexpr auto S { SimEngine::SRPIC }; - - SRPICEngine(const SimulationParams& params) : base_t { params } {} - - ~SRPICEngine() = default; - - void step_forward(timer::Timers& timers, domain_t& dom) override { - const auto fieldsolver_enabled = m_params.template get( - "algorithms.fieldsolver.enable"); - const auto deposit_enabled = m_params.template get( - "algorithms.deposit.enable"); - const auto clear_interval = m_params.template get( - "particles.clear_interval"); - - if (step == 0) { - // communicate fields and apply BCs on the first timestep - m_metadomain.CommunicateFields(dom, Comm::B | Comm::E); - FieldBoundaries(dom, BC::B | BC::E); - ParticleInjector(dom); - } - - if (fieldsolver_enabled) { - timers.start("FieldSolver"); - Faraday(dom, HALF); - timers.stop("FieldSolver"); - - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B); - timers.stop("Communications"); - - timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::B); - timers.stop("FieldBoundaries"); - Kokkos::fence(); - } - - { - timers.start("ParticlePusher"); - ParticlePush(dom); - timers.stop("ParticlePusher"); - - if (deposit_enabled) { - timers.start("CurrentDeposit"); - Kokkos::deep_copy(dom.fields.cur, ZERO); - CurrentsDeposit(dom); - timers.stop("CurrentDeposit"); - - timers.start("Communications"); - m_metadomain.SynchronizeFields(dom, Comm::J); - m_metadomain.CommunicateFields(dom, Comm::J); - timers.stop("Communications"); - - timers.start("CurrentFiltering"); - CurrentsFilter(dom); - timers.stop("CurrentFiltering"); - } - - timers.start("Communications"); - m_metadomain.CommunicateParticles(dom); - timers.stop("Communications"); - } - - if (fieldsolver_enabled) { - timers.start("FieldSolver"); - Faraday(dom, HALF); - timers.stop("FieldSolver"); - - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B); - timers.stop("Communications"); - - timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::B); - timers.stop("FieldBoundaries"); - - timers.start("FieldSolver"); - Ampere(dom, ONE); - timers.stop("FieldSolver"); - - if (deposit_enabled) { - timers.start("FieldSolver"); - CurrentsAmpere(dom); - timers.stop("FieldSolver"); - } - - timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E | Comm::J); - timers.stop("Communications"); - - timers.start("FieldBoundaries"); - FieldBoundaries(dom, BC::E); - timers.stop("FieldBoundaries"); - } - - { - timers.start("Injector"); - ParticleInjector(dom); - timers.stop("Injector"); - } - - if (clear_interval > 0 and step % clear_interval == 0 and step > 0) { - timers.start("PrtlClear"); - m_metadomain.RemoveDeadParticles(dom); - timers.stop("PrtlClear"); - } - } - - /* algorithm substeps --------------------------------------------------- */ - void Faraday(domain_t& domain, real_t fraction = ONE) { - logger::Checkpoint("Launching Faraday kernel", HERE); - const auto dT = fraction * - m_params.template get( - "algorithms.timestep.correction") * - dt; - if constexpr (M::CoordType == Coord::Cart) { - // minkowski case - const auto dx = math::sqrt(domain.mesh.metric.template h_<1, 1>({})); - const auto deltax = m_params.template get( - "algorithms.fieldsolver.delta_x"); - const auto deltay = m_params.template get( - "algorithms.fieldsolver.delta_y"); - const auto betaxy = m_params.template get( - "algorithms.fieldsolver.beta_xy"); - const auto betayx = m_params.template get( - "algorithms.fieldsolver.beta_yx"); - const auto deltaz = m_params.template get( - "algorithms.fieldsolver.delta_z"); - const auto betaxz = m_params.template get( - "algorithms.fieldsolver.beta_xz"); - const auto betazx = m_params.template get( - "algorithms.fieldsolver.beta_zx"); - const auto betayz = m_params.template get( - "algorithms.fieldsolver.beta_yz"); - const auto betazy = m_params.template get( - "algorithms.fieldsolver.beta_zy"); - real_t coeff1, coeff2; - if constexpr (M::Dim == Dim::_2D) { - coeff1 = dT / SQR(dx); - coeff2 = dT; - } else { - coeff1 = dT / dx; - coeff2 = ZERO; - } - Kokkos::parallel_for("Faraday", - domain.mesh.rangeActiveCells(), - kernel::mink::Faraday_kernel(domain.fields.em, - coeff1, - coeff2, - deltax, - deltay, - betaxy, - betayx, - deltaz, - betaxz, - betazx, - betayz, - betazy)); - } else { - Kokkos::parallel_for("Faraday", - domain.mesh.rangeActiveCells(), - kernel::sr::Faraday_kernel(domain.fields.em, - domain.mesh.metric, - dT, - domain.mesh.flds_bc())); - } - } - - void Ampere(domain_t& domain, real_t fraction = ONE) { - logger::Checkpoint("Launching Ampere kernel", HERE); - const auto dT = fraction * - m_params.template get( - "algorithms.timestep.correction") * - dt; - auto range = range_with_axis_BCs(domain); - if constexpr (M::CoordType == Coord::Cart) { - // minkowski case - const auto dx = math::sqrt(domain.mesh.metric.template h_<1, 1>({})); - real_t coeff1, coeff2; - if constexpr (M::Dim == Dim::_2D) { - coeff1 = dT / SQR(dx); - coeff2 = dT; - } else { - coeff1 = dT / dx; - coeff2 = ZERO; - } - - Kokkos::parallel_for( - "Ampere", - range, - kernel::mink::Ampere_kernel(domain.fields.em, coeff1, coeff2)); - } else { - const auto ni2 = domain.mesh.n_active(in::x2); - Kokkos::parallel_for("Ampere", - range, - kernel::sr::Ampere_kernel(domain.fields.em, - domain.mesh.metric, - dT, - ni2, - domain.mesh.flds_bc())); - } - } - - void ParticlePush(domain_t& domain) { - real_t gx1 { ZERO }, gx2 { ZERO }, gx3 { ZERO }, ds { ZERO }; - real_t x_surf { ZERO }; - bool has_atmosphere = false; - for (auto& direction : dir::Directions::orth) { - if (m_metadomain.mesh().prtl_bc_in(direction) == PrtlBC::ATMOSPHERE) { - raise::ErrorIf(has_atmosphere, - "Only one direction is allowed to have atm boundaries", - HERE); - has_atmosphere = true; - const auto g = m_params.template get( - "grid.boundaries.atmosphere.g"); - ds = m_params.template get("grid.boundaries.atmosphere.ds"); - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - if (dim == in::x1) { - gx1 = sign > 0 ? g : -g; - gx2 = ZERO; - gx3 = ZERO; - } else if (dim == in::x2) { - gx1 = ZERO; - gx2 = sign > 0 ? g : -g; - gx3 = ZERO; - } else if (dim == in::x3) { - gx1 = ZERO; - gx2 = ZERO; - gx3 = sign > 0 ? g : -g; - } else { - raise::Error("Invalid dimension", HERE); - } - if (sign > 0) { - x_surf = xg_min; - } else { - x_surf = xg_max; - } - } - } - for (auto& species : domain.species) { - if ((species.pusher() == PrtlPusher::NONE) or (species.npart() == 0)) { - continue; - } - species.set_unsorted(); - logger::Checkpoint( - fmt::format("Launching particle pusher kernel for %d [%s] : %lu", - species.index(), - species.label().c_str(), - species.npart()), - HERE); - const auto q_ovr_m = species.mass() > ZERO - ? species.charge() / species.mass() - : ZERO; - // coeff = q / m (dt / 2) omegaB0 - const auto coeff = q_ovr_m * HALF * dt * - m_params.template get("scales.omegaB0"); - PrtlPusher::type pusher; - if (species.pusher() == PrtlPusher::PHOTON) { - pusher = PrtlPusher::PHOTON; - } else if (species.pusher() == PrtlPusher::BORIS) { - pusher = PrtlPusher::BORIS; - } else if (species.pusher() == PrtlPusher::VAY) { - pusher = PrtlPusher::VAY; - } else { - raise::Fatal("Invalid particle pusher", HERE); - } - const auto cooling = species.cooling(); - - // coefficients to be forwarded to the dispatcher - // gca - const auto has_gca = species.use_gca(); - const auto gca_larmor_max = has_gca ? m_params.template get( - "algorithms.gca.larmor_max") - : ZERO; - const auto gca_eovrb_max = has_gca ? m_params.template get( - "algorithms.gca.e_ovr_b_max") - : ZERO; - // cooling - const auto has_synchrotron = (cooling == Cooling::SYNCHROTRON); - const auto has_compton = (cooling == Cooling::COMPTON); - const auto sync_grad = has_synchrotron - ? m_params.template get( - "algorithms.synchrotron.gamma_rad") - : ZERO; - const auto sync_coeff = has_synchrotron - ? (real_t)(0.1) * dt * - m_params.template get( - "scales.omegaB0") / - (SQR(sync_grad) * species.mass()) - : ZERO; - const auto comp_grad = has_compton ? m_params.template get( - "algorithms.compton.gamma_rad") - : ZERO; - const auto comp_coeff = has_compton ? (real_t)(0.1) * dt * - m_params.template get( - "scales.omegaB0") / - (SQR(comp_grad) * species.mass()) - : ZERO; - // toggle to indicate whether pgen defines the external force - bool has_extforce = false; - if constexpr (traits::has_member::value) { - has_extforce = true; - // toggle to indicate whether the ext force applies to current species - if (traits::has_member::value) { - has_extforce &= std::find(m_pgen.ext_force.species.begin(), - m_pgen.ext_force.species.end(), - species.index()) != - m_pgen.ext_force.species.end(); - } - } - - kernel::sr::CoolingTags cooling_tags = 0; - if (cooling == Cooling::SYNCHROTRON) { - cooling_tags = kernel::sr::Cooling::Synchrotron; - } - if (cooling == Cooling::COMPTON) { - cooling_tags = kernel::sr::Cooling::Compton; - } - // clang-format off - if (not has_atmosphere and not has_extforce) { - Kokkos::parallel_for( - "ParticlePusher", - species.rangeActiveParticles(), - kernel::sr::Pusher_kernel( - pusher, has_gca, false, - cooling_tags, - domain.fields.em, - species.index(), - species.i1, species.i2, species.i3, - species.i1_prev, species.i2_prev, species.i3_prev, - species.dx1, species.dx2, species.dx3, - species.dx1_prev, species.dx2_prev, species.dx3_prev, - species.ux1, species.ux2, species.ux3, - species.phi, species.tag, - domain.mesh.metric, - time, coeff, dt, - domain.mesh.n_active(in::x1), - domain.mesh.n_active(in::x2), - domain.mesh.n_active(in::x3), - domain.mesh.prtl_bc(), - gca_larmor_max, gca_eovrb_max, sync_coeff, comp_coeff - )); - } else if (has_atmosphere and not has_extforce) { - const auto force = - kernel::sr::Force { - {gx1, gx2, gx3}, - x_surf, - ds - }; - Kokkos::parallel_for( - "ParticlePusher", - species.rangeActiveParticles(), - kernel::sr::Pusher_kernel( - pusher, has_gca, false, - cooling_tags, - domain.fields.em, - species.index(), - species.i1, species.i2, species.i3, - species.i1_prev, species.i2_prev, species.i3_prev, - species.dx1, species.dx2, species.dx3, - species.dx1_prev, species.dx2_prev, species.dx3_prev, - species.ux1, species.ux2, species.ux3, - species.phi, species.tag, - domain.mesh.metric, - force, - time, coeff, dt, - domain.mesh.n_active(in::x1), - domain.mesh.n_active(in::x2), - domain.mesh.n_active(in::x3), - domain.mesh.prtl_bc(), - gca_larmor_max, gca_eovrb_max, sync_coeff, comp_coeff - )); - } else if (not has_atmosphere and has_extforce) { - if constexpr (traits::has_member::value) { - const auto force = - kernel::sr::Force { - m_pgen.ext_force - }; - Kokkos::parallel_for( - "ParticlePusher", - species.rangeActiveParticles(), - kernel::sr::Pusher_kernel( - pusher, has_gca, true, - cooling_tags, - domain.fields.em, - species.index(), - species.i1, species.i2, species.i3, - species.i1_prev, species.i2_prev, species.i3_prev, - species.dx1, species.dx2, species.dx3, - species.dx1_prev, species.dx2_prev, species.dx3_prev, - species.ux1, species.ux2, species.ux3, - species.phi, species.tag, - domain.mesh.metric, - force, - time, coeff, dt, - domain.mesh.n_active(in::x1), - domain.mesh.n_active(in::x2), - domain.mesh.n_active(in::x3), - domain.mesh.prtl_bc(), - gca_larmor_max, gca_eovrb_max, sync_coeff, comp_coeff - )); - } else { - raise::Error("External force not implemented", HERE); - } - } else { // has_atmosphere and has_extforce - if constexpr (traits::has_member::value) { - const auto force = - kernel::sr::Force { - m_pgen.ext_force, {gx1, gx2, gx3}, x_surf, ds - }; - Kokkos::parallel_for( - "ParticlePusher", - species.rangeActiveParticles(), - kernel::sr::Pusher_kernel( - pusher, has_gca, true, - cooling_tags, - domain.fields.em, - species.index(), - species.i1, species.i2, species.i3, - species.i1_prev, species.i2_prev, species.i3_prev, - species.dx1, species.dx2, species.dx3, - species.dx1_prev, species.dx2_prev, species.dx3_prev, - species.ux1, species.ux2, species.ux3, - species.phi, species.tag, - domain.mesh.metric, - force, - time, coeff, dt, - domain.mesh.n_active(in::x1), - domain.mesh.n_active(in::x2), - domain.mesh.n_active(in::x3), - domain.mesh.prtl_bc(), - gca_larmor_max, gca_eovrb_max, sync_coeff, comp_coeff - )); - } else { - raise::Error("External force not implemented", HERE); - } - } - // clang-format on - } - } - - void ParticleInjector(domain_t& domain, InjTags tags = Inj::None) { - for (auto& direction : dir::Directions::orth) { - if (m_metadomain.mesh().prtl_bc_in(direction) == PrtlBC::ATMOSPHERE) { - AtmosphereParticlesIn(direction, domain, tags); - } - } - } - - template - void deposit_with(const Particles& species, - const M& metric, - const scatter_ndfield_t& scatter_cur, - real_t dt) { - // clang-format off - Kokkos::parallel_for("CurrentsDeposit", - species.rangeActiveParticles(), - kernel::DepositCurrents_kernel( - scatter_cur, - species.i1, species.i2, species.i3, - species.i1_prev, species.i2_prev, species.i3_prev, - species.dx1, species.dx2, species.dx3, - species.dx1_prev, species.dx2_prev, species.dx3_prev, - species.ux1, species.ux2, species.ux3, - species.phi, species.weight, species.tag, - metric, (real_t)(species.charge()), dt)); - // clang-format on - } - - void CurrentsDeposit(domain_t& domain) { - auto scatter_cur = Kokkos::Experimental::create_scatter_view( - domain.fields.cur); - auto shape_order = m_params.template get("algorithms.deposit.order"); - for (auto& species : domain.species) { - if ((species.pusher() == PrtlPusher::NONE) or (species.npart() == 0) or - cmp::AlmostZero_host(species.charge())) { - continue; - } - logger::Checkpoint( - fmt::format("Launching currents deposit kernel for %d [%s] : %lu %f", - species.index(), - species.label().c_str(), - species.npart(), - (double)species.charge()), - HERE); - - deposit_with(species, domain.mesh.metric, scatter_cur, dt); - } - Kokkos::Experimental::contribute(domain.fields.cur, scatter_cur); - } - - void CurrentsAmpere(domain_t& domain) { - logger::Checkpoint("Launching Ampere kernel for adding currents", HERE); - const auto q0 = m_params.template get("scales.q0"); - const auto n0 = m_params.template get("scales.n0"); - const auto B0 = m_params.template get("scales.B0"); - if constexpr (M::CoordType == Coord::Cart) { - // minkowski case - const auto V0 = m_params.template get("scales.V0"); - const auto ppc0 = m_params.template get("particles.ppc0"); - const auto coeff = -dt * q0 / (B0 * V0); - if constexpr ( - traits::has_member::value) { - const std::vector xmin { domain.mesh.extent(in::x1).first, - domain.mesh.extent(in::x2).first, - domain.mesh.extent(in::x3).first }; - const auto ext_current = m_pgen.ext_current; - const auto dx = domain.mesh.metric.template sqrt_h_<1, 1>({}); - // clang-format off - Kokkos::parallel_for( - "Ampere", - domain.mesh.rangeActiveCells(), - kernel::mink::CurrentsAmpere_kernel( - domain.fields.em, domain.fields.cur, - coeff, ppc0, ext_current, xmin, dx)); - // clang-format on - } else { - Kokkos::parallel_for( - "Ampere", - domain.mesh.rangeActiveCells(), - kernel::mink::CurrentsAmpere_kernel(domain.fields.em, - domain.fields.cur, - coeff, - ppc0)); - } - } else { - // non-minkowski - const auto coeff = -dt * q0 * n0 / B0; - auto range = range_with_axis_BCs(domain); - const auto ni2 = domain.mesh.n_active(in::x2); - Kokkos::parallel_for( - "Ampere", - range, - kernel::sr::CurrentsAmpere_kernel(domain.fields.em, - domain.fields.cur, - domain.mesh.metric, - coeff, - ONE / n0, - ni2, - domain.mesh.flds_bc())); - } - } - - void CurrentsFilter(domain_t& domain) { - logger::Checkpoint("Launching currents filtering kernels", HERE); - auto range = range_with_axis_BCs(domain); - const auto nfilter = m_params.template get( - "algorithms.current_filters"); - tuple_t size; - if constexpr (M::Dim == Dim::_1D || M::Dim == Dim::_2D || M::Dim == Dim::_3D) { - size[0] = domain.mesh.n_active(in::x1); - } - if constexpr (M::Dim == Dim::_2D || M::Dim == Dim::_3D) { - size[1] = domain.mesh.n_active(in::x2); - } - if constexpr (M::Dim == Dim::_3D) { - size[2] = domain.mesh.n_active(in::x3); - } - // !TODO: this needs to be done more efficiently - for (auto i { 0u }; i < nfilter; ++i) { - Kokkos::deep_copy(domain.fields.buff, domain.fields.cur); - Kokkos::parallel_for("CurrentsFilter", - range, - kernel::DigitalFilter_kernel( - domain.fields.cur, - domain.fields.buff, - size, - domain.mesh.flds_bc())); - m_metadomain.CommunicateFields(domain, Comm::J); - } - } - - void FieldBoundaries(domain_t& domain, BCTags tags) { - for (auto& direction : dir::Directions::orth) { - if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::MATCH) { - MatchFieldsIn(direction, domain, tags); - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::AXIS) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { - AxisFieldsIn(direction, domain, tags); - } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::ATMOSPHERE) { - AtmosphereFieldsIn(direction, domain, tags); - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::FIXED) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::FIXED) { - FixedFieldsIn(direction, domain, tags); - } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CONDUCTOR) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { - PerfectConductorFieldsIn(direction, domain, tags); - } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::CUSTOM) { - if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { - CustomFieldsIn(direction, domain, tags); - } - } else if (m_metadomain.mesh().flds_bc_in(direction) == FldsBC::HORIZON) { - raise::Error("HORIZON BCs only applicable for GR", HERE); - } - } // loop over directions - } - - void MatchFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - /** - * matching boundaries - */ - const auto ds_array = m_params.template get>( - "grid.boundaries.match.ds"); - const auto dim = direction.get_dim(); - real_t xg_min, xg_max, xg_edge; - auto sign = direction.get_sign(); - real_t ds; - if (sign > 0) { // + direction - ds = ds_array[(short)dim].second; - xg_max = m_metadomain.mesh().extent(dim).second; - xg_min = xg_max - ds; - xg_edge = xg_max; - } else { // - direction - ds = ds_array[(short)dim].first; - xg_min = m_metadomain.mesh().extent(dim).first; - xg_max = xg_min + ds; - xg_edge = xg_min; - } - boundaries_t box; - boundaries_t incl_ghosts; - for (dim_t d { 0 }; d < M::Dim; ++d) { - if (d == static_cast(dim)) { - box.push_back({ xg_min, xg_max }); - if (sign > 0) { - incl_ghosts.push_back({ false, true }); - } else { - incl_ghosts.push_back({ true, false }); - } - } else { - box.push_back(Range::All); - incl_ghosts.push_back({ true, true }); - } - } - if (not domain.mesh.Intersects(box)) { - return; - } - const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; - - for (auto d { 0u }; d < M::Dim; ++d) { - range_min[d] = intersect_range[d].first; - range_max[d] = intersect_range[d].second; - } - - if (dim == in::x1) { - if constexpr ( - traits::has_member::value) { - auto match_fields = m_pgen.MatchFields(time); - call_match_fields(domain.fields.em, - domain.mesh.flds_bc(), - match_fields, - domain.mesh.metric, - xg_edge, - ds, - tags, - range_min, - range_max); - } else if constexpr ( - traits::has_member::value) { - auto match_fields = m_pgen.MatchFieldsInX1(time); - call_match_fields(domain.fields.em, - domain.mesh.flds_bc(), - match_fields, - domain.mesh.metric, - xg_edge, - ds, - tags, - range_min, - range_max); - } - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - if constexpr ( - traits::has_member::value) { - auto match_fields = m_pgen.MatchFields(time); - call_match_fields(domain.fields.em, - domain.mesh.flds_bc(), - match_fields, - domain.mesh.metric, - xg_edge, - ds, - tags, - range_min, - range_max); - } else if constexpr ( - traits::has_member::value) { - auto match_fields = m_pgen.MatchFieldsInX2(time); - call_match_fields(domain.fields.em, - domain.mesh.flds_bc(), - match_fields, - domain.mesh.metric, - xg_edge, - ds, - tags, - range_min, - range_max); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - if constexpr ( - traits::has_member::value) { - auto match_fields = m_pgen.MatchFields(time); - call_match_fields(domain.fields.em, - domain.mesh.flds_bc(), - match_fields, - domain.mesh.metric, - xg_edge, - ds, - tags, - range_min, - range_max); - } else if constexpr ( - traits::has_member::value) { - auto match_fields = m_pgen.MatchFieldsInX3(time); - call_match_fields(domain.fields.em, - domain.mesh.flds_bc(), - match_fields, - domain.mesh.metric, - xg_edge, - ds, - tags, - range_min, - range_max); - } - } - } else { - raise::Error("Invalid dimension", HERE); - } - } - - void AxisFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - /** - * axis boundaries - */ - if constexpr (M::CoordType != Coord::Cart) { - raise::ErrorIf(direction.get_dim() != in::x2, - "Invalid axis direction, should be x2", - HERE); - const auto i2_min = domain.mesh.i_min(in::x2); - const auto i2_max = domain.mesh.i_max(in::x2); - if (direction.get_sign() < 0) { - Kokkos::parallel_for( - "AxisBCFields", - domain.mesh.n_all(in::x1), - kernel::bc::AxisBoundaries_kernel(domain.fields.em, - i2_min, - tags)); - } else { - Kokkos::parallel_for( - "AxisBCFields", - domain.mesh.n_all(in::x1), - kernel::bc::AxisBoundaries_kernel(domain.fields.em, - i2_max, - tags)); - } - } else { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Invalid coordinate type for axis BCs", HERE); - } - } - - void FixedFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - /** - * fixed field boundaries - */ - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, - "Fixed BCs only implemented for x1 in " - "non-cartesian coordinates", - HERE); - em normal_b_comp, tang_e_comp1, tang_e_comp2; - if (dim == in::x1) { - normal_b_comp = em::bx1; - tang_e_comp1 = em::ex2; - tang_e_comp2 = em::ex3; - } else if (dim == in::x2) { - normal_b_comp = em::bx2; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex3; - } else if (dim == in::x3) { - normal_b_comp = em::bx3; - tang_e_comp1 = em::ex1; - tang_e_comp2 = em::ex2; - } else { - raise::Error("Invalid dimension", HERE); - } - std::vector xi_min, xi_max; - const std::vector all_dirs { in::x1, in::x2, in::x3 }; - for (dim_t d { 0u }; d < M::Dim; ++d) { - const auto dd = all_dirs[d]; - if (dim == dd) { - if (sign > 0) { // + direction - xi_min.push_back(domain.mesh.n_all(dd) - N_GHOSTS); - xi_max.push_back(domain.mesh.n_all(dd)); - } else { // - direction - xi_min.push_back(0); - xi_max.push_back(N_GHOSTS); - } - } else { - xi_min.push_back(0); - xi_max.push_back(domain.mesh.n_all(dd)); - } - } - raise::ErrorIf(xi_min.size() != xi_max.size() or - xi_min.size() != static_cast(M::Dim), - "Invalid range size", - HERE); - std::vector comps; - if (tags & BC::E) { - comps.push_back(tang_e_comp1); - comps.push_back(tang_e_comp2); - } - if (tags & BC::B) { - comps.push_back(normal_b_comp); - } - if constexpr (traits::has_member::value) { - raise::Error("Non-const fixed fields not implemented", HERE); - } else if constexpr ( - traits::has_member::value) { - for (const auto& comp : comps) { - auto value = ZERO; - bool shouldset = false; - if constexpr ( - traits::has_member::value) { - // if fix field function present, read from it - const auto newset = m_pgen.FixFieldsConst( - (bc_in)(sign * ((short)dim + 1)), - (em)comp); - value = newset.first; - shouldset = newset.second; - } - if (shouldset) { - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(xi_min[0], xi_max[0]), - comp), - value); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(xi_min[0], xi_max[0]), - std::make_pair(xi_min[1], xi_max[1]), - comp), - value); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.em, - std::make_pair(xi_min[0], xi_max[0]), - std::make_pair(xi_min[1], xi_max[1]), - std::make_pair(xi_min[2], xi_max[2]), - comp), - value); - } else { - raise::Error("Invalid dimension", HERE); - } - } - } - } else { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Fixed fields not present (both const and non-const)", HERE); - } - } - - void PerfectConductorFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - /** - * perfect conductor field boundaries - */ - if constexpr (M::CoordType != Coord::Cart) { - (void)direction; - (void)domain; - (void)tags; - raise::Error( - "Perfect conductor BCs only applicable to cartesian coordinates", - HERE); - } else { - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - - std::vector xi_min, xi_max; - - const std::vector all_dirs { in::x1, in::x2, in::x3 }; - - for (auto d { 0u }; d < M::Dim; ++d) { - const auto dd = all_dirs[d]; - if (dim == dd) { - xi_min.push_back(0); - xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); - } else { - xi_min.push_back(0); - xi_max.push_back(domain.mesh.n_all(dd)); - } - } - raise::ErrorIf(xi_min.size() != xi_max.size() or - xi_min.size() != static_cast(M::Dim), - "Invalid range size", - HERE); - - range_t range; - if constexpr (M::Dim == Dim::_1D) { - range = CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); - } else if constexpr (M::Dim == Dim::_2D) { - range = CreateRangePolicy({ xi_min[0], xi_min[1] }, - { xi_max[0], xi_max[1] }); - } else if constexpr (M::Dim == Dim::_3D) { - range = CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, - { xi_max[0], xi_max[1], xi_max[2] }); - } else { - raise::Error("Invalid dimension", HERE); - } - std::size_t i_edge; - if (sign > 0) { - i_edge = domain.mesh.i_max(dim); - } else { - i_edge = domain.mesh.i_min(dim); - } - - if (dim == in::x1) { - if (sign > 0) { - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - i_edge, - tags)); - } else { - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - i_edge, - tags)); - } - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - i_edge, - tags)); - } else { - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - i_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else { - if constexpr (M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - i_edge, - tags)); - } else { - Kokkos::parallel_for( - "ConductorFields", - range, - kernel::bc::ConductorBoundaries_kernel( - domain.fields.em, - i_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } - } - } - - void AtmosphereFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - /** - * atmosphere field boundaries - */ - if constexpr (traits::has_member::value) { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - const auto dd = static_cast(dim); - boundaries_t box; - boundaries_t incl_ghosts; - for (auto d { 0u }; d < M::Dim; ++d) { - if (d == dd) { - box.push_back({ xg_min, xg_max }); - if (sign > 0) { - incl_ghosts.push_back({ false, true }); - } else { - incl_ghosts.push_back({ true, false }); - } - } else { - box.push_back(Range::All); - incl_ghosts.push_back({ true, true }); - } - } - if (not domain.mesh.Intersects(box)) { - return; - } - const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); - tuple_t range_min { 0 }; - tuple_t range_max { 0 }; - - for (auto d { 0u }; d < M::Dim; ++d) { - range_min[d] = intersect_range[d].first; - range_max[d] = intersect_range[d].second; - } - auto atm_fields = m_pgen.AtmFields(time); - std::size_t il_edge; - if (sign > 0) { - il_edge = range_min[dd] - N_GHOSTS; - } else { - il_edge = range_max[dd] - 1 - N_GHOSTS; - } - const auto range = CreateRangePolicy(range_min, range_max); - if (dim == in::x1) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - if (sign > 0) { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } else { - Kokkos::parallel_for( - "AtmosphereBCFields", - range, - kernel::bc::EnforcedBoundaries_kernel( - domain.fields.em, - atm_fields, - domain.mesh.metric, - il_edge, - tags)); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else { - raise::Error("Invalid dimension", HERE); - } - } else { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Atm fields not implemented in PGEN for atmosphere BCs", HERE); - } - } - - void CustomFieldsIn(dir::direction_t direction, - domain_t& domain, - BCTags tags) { - (void)direction; - (void)domain; - (void)tags; - raise::Error("Custom boundaries not implemented", HERE); - // if constexpr ( - // traits::has_member::value) { - // const auto [box, custom_fields] = m_pgen.CustomFields(time); - // if (domain.mesh.Intersects(box)) { - // } - // - // } else { - // raise::Error("Custom boundaries not implemented", HERE); - // } - } - - void AtmosphereParticlesIn(const dir::direction_t& direction, - domain_t& domain, - InjTags tags) { - const auto [sign, dim, xg_min, xg_max] = get_atm_extent(direction); - - const auto x_surf = sign > 0 ? xg_min : xg_max; - const auto ds = m_params.template get( - "grid.boundaries.atmosphere.ds"); - const auto temp = m_params.template get( - "grid.boundaries.atmosphere.temperature"); - const auto height = m_params.template get( - "grid.boundaries.atmosphere.height"); - const auto species = m_params.template get>( - "grid.boundaries.atmosphere.species"); - const auto nmax = m_params.template get( - "grid.boundaries.atmosphere.density"); - - Kokkos::deep_copy(domain.fields.bckp, ZERO); - auto scatter_bckp = Kokkos::Experimental::create_scatter_view( - domain.fields.bckp); - const auto use_weights = M::CoordType != Coord::Cart; - const auto ni2 = domain.mesh.n_active(in::x2); - const auto inv_n0 = ONE / m_params.template get("scales.n0"); - - // compute the density of the two species - if (tags & Inj::AssumeEmpty) { - if constexpr (M::Dim == Dim::_1D) { - Kokkos::deep_copy( - Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), - ZERO); - } else if constexpr (M::Dim == Dim::_2D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } else if constexpr (M::Dim == Dim::_3D) { - Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, - Kokkos::ALL, - Kokkos::ALL, - Kokkos::ALL, - std::make_pair(0, 1)), - ZERO); - } - } else { - for (const auto& sp : - std::vector { species.first, species.second }) { - auto& prtl_spec = domain.species[sp - 1]; - if (prtl_spec.npart() == 0) { - continue; - } - // clang-format off - Kokkos::parallel_for( - "ComputeMoments", - prtl_spec.rangeActiveParticles(), - kernel::ParticleMoments_kernel( - {}, scatter_bckp, 0, - prtl_spec.i1, prtl_spec.i2, prtl_spec.i3, - prtl_spec.dx1, prtl_spec.dx2, prtl_spec.dx3, - prtl_spec.ux1, prtl_spec.ux2, prtl_spec.ux3, - prtl_spec.phi, prtl_spec.weight, prtl_spec.tag, - prtl_spec.mass(), prtl_spec.charge(), - use_weights, - domain.mesh.metric, domain.mesh.flds_bc(), - ni2, inv_n0, 0)); - // clang-format on - prtl_spec.set_unsorted(); - } - Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); - m_metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); - } - - const auto maxwellian = arch::Maxwellian { domain.mesh.metric, - domain.random_pool(), - temp }; - - if (dim == in::x1) { - if (sign > 0) { - auto target_density = - arch::AtmosphereDensityProfile { - nmax, - height, - x_surf, - ds - }; - const auto spatial_dist = arch::Replenish { - domain.mesh.metric, - domain.fields.bckp, - 0, - target_density, - nmax - }; - arch::InjectNonUniform( - m_params, - domain, - { species.first, species.second }, - { maxwellian, maxwellian }, - spatial_dist, - nmax, - use_weights); - } else { - auto target_density = - arch::AtmosphereDensityProfile { - nmax, - height, - x_surf, - ds - }; - const auto spatial_dist = arch::Replenish { - domain.mesh.metric, - domain.fields.bckp, - 0, - target_density, - nmax - }; - arch::InjectNonUniform( - m_params, - domain, - { species.first, species.second }, - { maxwellian, maxwellian }, - spatial_dist, - nmax, - use_weights); - } - } else if (dim == in::x2) { - if (sign > 0) { - auto target_density = - arch::AtmosphereDensityProfile { - nmax, - height, - x_surf, - ds - }; - const auto spatial_dist = arch::Replenish { - domain.mesh.metric, - domain.fields.bckp, - 0, - target_density, - nmax - }; - arch::InjectNonUniform( - m_params, - domain, - { species.first, species.second }, - { maxwellian, maxwellian }, - spatial_dist, - nmax, - use_weights); - } else { - auto target_density = - arch::AtmosphereDensityProfile { - nmax, - height, - x_surf, - ds - }; - const auto spatial_dist = arch::Replenish { - domain.mesh.metric, - domain.fields.bckp, - 0, - target_density, - nmax - }; - arch::InjectNonUniform( - m_params, - domain, - { species.first, species.second }, - { maxwellian, maxwellian }, - spatial_dist, - nmax, - use_weights); - } - } else if (dim == in::x3) { - if (sign > 0) { - auto target_density = - arch::AtmosphereDensityProfile { - nmax, - height, - x_surf, - ds - }; - const auto spatial_dist = arch::Replenish { - domain.mesh.metric, - domain.fields.bckp, - 0, - target_density, - nmax - }; - arch::InjectNonUniform( - m_params, - domain, - { species.first, species.second }, - { maxwellian, maxwellian }, - spatial_dist, - nmax, - use_weights); - } else { - auto target_density = - arch::AtmosphereDensityProfile { - nmax, - height, - x_surf, - ds - }; - const auto spatial_dist = arch::Replenish { - domain.mesh.metric, - domain.fields.bckp, - 0, - target_density, - nmax - }; - arch::InjectNonUniform( - m_params, - domain, - { species.first, species.second }, - { maxwellian, maxwellian }, - spatial_dist, - nmax, - use_weights); - } - } else { - raise::Error("Invalid dimension", HERE); - } - return; - } - - private: - /** - * @brief Get the buffer region of the atmosphere and the direction - * @param direction direction in which the atmosphere is applied - * @return tuple: [sign of the direction, the direction (as in::), the min and max extent - * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself - * @note i.e. - * - * fields set particles injected - * ghost zone | | - * v v v - * |....|...........|*******************..... -> x1 - * ^ ^ - * xg_min xg_max - * | | | - * |<-- buffer -->|<-- atmosphere -->| - * - * in this case the function returns { -1, in::x1, xg_min, xg_max } - */ - auto get_atm_extent(dir::direction_t direction) const - -> std::tuple { - const auto sign = direction.get_sign(); - const auto dim = direction.get_dim(); - const auto min_buff = m_params.template get( - "algorithms.current_filters") + - 2; - const auto buffer_ncells = min_buff > 5 ? min_buff : 5; - if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) { - raise::Error("For non-cartesian coordinates atmosphere BCs is " - "possible only in -x1 (@ rmin)", - HERE); - } - real_t xg_min { ZERO }, xg_max { ZERO }; - ncells_t ig_min, ig_max; - if (sign > 0) { // + direction - ig_min = m_metadomain.mesh().n_active(dim) - buffer_ncells; - ig_max = m_metadomain.mesh().n_active(dim); - } else { // - direction - ig_min = 0; - ig_max = buffer_ncells; - } - - if (dim == in::x1) { - xg_min = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<1, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else if (dim == in::x2) { - if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { - xg_min = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<2, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else { - raise::Error("Invalid dimension", HERE); - } - } else if (dim == in::x3) { - if constexpr (M::Dim == Dim::_3D) { - xg_min = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_min)); - xg_max = m_metadomain.mesh().metric.template convert<3, Crd::Cd, Crd::Ph>( - static_cast(ig_max)); - } else { - raise::Error("Invalid dimension", HERE); - } - } else { - raise::Error("Invalid dimension", HERE); - } - return { sign, dim, xg_min, xg_max }; - } - - auto range_with_axis_BCs(const domain_t& domain) -> range_t { - auto range = domain.mesh.rangeActiveCells(); - if constexpr (M::CoordType != Coord::Cart) { - /** - * @brief taking one extra cell in the x2 direction if AXIS BCs - */ - if constexpr (M::Dim == Dim::_2D) { - if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { - range = CreateRangePolicy( - { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, - { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); - } - } else if constexpr (M::Dim == Dim::_3D) { - if (domain.mesh.flds_bc_in({ 0, +1, 0 }) == FldsBC::AXIS) { - range = CreateRangePolicy({ domain.mesh.i_min(in::x1), - domain.mesh.i_min(in::x2), - domain.mesh.i_min(in::x3) }, - { domain.mesh.i_max(in::x1), - domain.mesh.i_max(in::x2) + 1, - domain.mesh.i_max(in::x3) }); - } - } - } - return range; - } - - template - void call_match_fields(ndfield_t& fields, - const boundaries_t& boundaries, - const T& match_fields, - const M& metric, - real_t xg_edge, - real_t ds, - BCTags tags, - tuple_t& range_min, - tuple_t& range_max) { - Kokkos::parallel_for( - "MatchFields", - CreateRangePolicy(range_min, range_max), - kernel::bc::MatchBoundaries_kernel(fields, - match_fields, - metric, - xg_edge, - ds, - tags, - boundaries)); - } - }; - -} // namespace ntt - -#endif // ENGINES_SRPIC_SRPIC_H diff --git a/src/engines/srpic/currents.h b/src/engines/srpic/currents.h new file mode 100644 index 000000000..5733875b4 --- /dev/null +++ b/src/engines/srpic/currents.h @@ -0,0 +1,116 @@ +#ifndef ENGINES_SRPIC_CURRENTS_H +#define ENGINES_SRPIC_CURRENTS_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/log.h" +#include "utils/param_container.h" + +#include "metrics/traits.h" + +#include "engines/srpic/utils.h" +#include "framework/domain/domain.h" +#include "framework/domain/metadomain.h" +#include "kernels/currents_deposit.hpp" +#include "kernels/digital_filter.hpp" + +namespace ntt { + namespace srpic { + + template + requires metric::traits::HasD && metric::traits::HasCoordType + void CallDepositKernel(const Particles& species, + const M& local_metric, + const scatter_ndfield_t& scatter_cur, + real_t dt) { + Kokkos::parallel_for("CurrentsDeposit", + species.rangeActiveParticles(), + kernel::DepositCurrents_kernel( + scatter_cur, + species.i1, + species.i2, + species.i3, + species.i1_prev, + species.i2_prev, + species.i3_prev, + species.dx1, + species.dx2, + species.dx3, + species.dx1_prev, + species.dx2_prev, + species.dx3_prev, + species.ux1, + species.ux2, + species.ux3, + species.phi, + species.weight, + species.tag, + local_metric, + (real_t)(species.charge()), + dt)); + } + + template + void CurrentsDeposit(Domain& domain, + const prm::Parameters& engine_params) { + const auto dt = engine_params.get("dt"); + Kokkos::deep_copy(domain.fields.cur, ZERO); + auto scatter_cur = Kokkos::Experimental::create_scatter_view( + domain.fields.cur); + for (auto& species : domain.species) { + if ((species.pusher() == ParticlePusher::NONE) or + (species.npart() == 0) or cmp::AlmostZero_host(species.charge())) { + continue; + } + logger::Checkpoint( + fmt::format("Launching currents deposit kernel for %d [%s] : %lu %f", + species.index(), + species.label().c_str(), + species.npart(), + (double)species.charge()), + HERE); + + CallDepositKernel(species, domain.mesh.metric, scatter_cur, dt); + } + Kokkos::Experimental::contribute(domain.fields.cur, scatter_cur); + } + + template + requires metric::traits::HasD && metric::traits::HasCoordType + void CurrentsFilter(Metadomain& metadomain, + Domain& domain, + const SimulationParams& params) { + logger::Checkpoint("Launching currents filtering kernels", HERE); + auto range = srpic::RangeWithAxisBCs(domain); + const auto nfilter = params.template get( + "algorithms.current_filters"); + tuple_t size; + if constexpr (M::Dim == Dim::_1D || M::Dim == Dim::_2D || M::Dim == Dim::_3D) { + size[0] = domain.mesh.n_active(in::x1); + } + if constexpr (M::Dim == Dim::_2D || M::Dim == Dim::_3D) { + size[1] = domain.mesh.n_active(in::x2); + } + if constexpr (M::Dim == Dim::_3D) { + size[2] = domain.mesh.n_active(in::x3); + } + // !TODO: this needs to be done more efficiently + for (auto i { 0u }; i < nfilter; ++i) { + Kokkos::deep_copy(domain.fields.buff, domain.fields.cur); + Kokkos::parallel_for("CurrentsFilter", + range, + kernel::DigitalFilter_kernel( + domain.fields.cur, + domain.fields.buff, + size, + domain.mesh.flds_bc())); + metadomain.CommunicateFields(domain, Comm::J); + } + } + + } // namespace srpic +} // namespace ntt + +#endif // ENGINES_SRPIC_CURRENTS_H diff --git a/src/engines/srpic/fields_bcs.h b/src/engines/srpic/fields_bcs.h new file mode 100644 index 000000000..a02909cdf --- /dev/null +++ b/src/engines/srpic/fields_bcs.h @@ -0,0 +1,663 @@ +#ifndef ENGINES_SRPIC_FIELDS_BCS_H +#define ENGINES_SRPIC_FIELDS_BCS_H + +#include "enums.h" +#include "global.h" + +#include "arch/directions.h" +#include "utils/numeric.h" + +#include "metrics/traits.h" + +#include "archetypes/traits.h" +#include "engines/srpic/utils.h" +#include "framework/domain/domain.h" +#include "framework/parameters/parameters.h" +#include "kernels/fields_bcs.hpp" + +namespace ntt { + namespace srpic { + + template + requires metric::traits::HasD + void CallMatchFields(ndfield_t& fields, + const boundaries_t& boundaries, + const T& match_fields, + const M& metric, + real_t xg_edge, + real_t ds, + BCTags tags, + tuple_t& range_min, + tuple_t& range_max) { + Kokkos::parallel_for( + "MatchFields", + CreateRangePolicy(range_min, range_max), + kernel::bc::MatchBoundaries_kernel(fields, + match_fields, + metric, + xg_edge, + ds, + tags, + boundaries)); + } + + template + requires metric::traits::HasD + void MatchFieldsIn(dir::direction_t direction, + Domain& domain, + const Grid& global_grid, + const PG& pgen, + const prm::Parameters& engine_params, + const SimulationParams& params, + BCTags tags) { + const auto time = engine_params.get("time"); + /** + * matching boundaries + */ + const auto ds_array = params.template get>( + "grid.boundaries.match.ds"); + const auto dim = direction.get_dim(); + real_t xg_min, xg_max, xg_edge; + auto sign = direction.get_sign(); + real_t ds; + if (sign > 0) { // + direction + ds = ds_array[(short)dim].second; + xg_max = global_grid.extent(dim).second; + xg_min = xg_max - ds; + xg_edge = xg_max; + } else { // - direction + ds = ds_array[(short)dim].first; + xg_min = global_grid.extent(dim).first; + xg_max = xg_min + ds; + xg_edge = xg_min; + } + boundaries_t box; + boundaries_t incl_ghosts; + for (dim_t d { 0 }; d < M::Dim; ++d) { + if (d == static_cast(dim)) { + box.push_back({ xg_min, xg_max }); + if (sign > 0) { + incl_ghosts.push_back({ false, true }); + } else { + incl_ghosts.push_back({ true, false }); + } + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (auto d { 0u }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + + if (dim == in::x1) { + if constexpr (arch::traits::pgen::HasMatchFields) { + auto match_fields = pgen.MatchFields(time); + CallMatchFields(domain.fields.em, + domain.mesh.flds_bc(), + match_fields, + domain.mesh.metric, + xg_edge, + ds, + tags, + range_min, + range_max); + } else if constexpr (arch::traits::pgen::HasMatchFieldsInX1) { + auto match_fields = pgen.MatchFieldsInX1(time); + CallMatchFields(domain.fields.em, + domain.mesh.flds_bc(), + match_fields, + domain.mesh.metric, + xg_edge, + ds, + tags, + range_min, + range_max); + } + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + if constexpr (arch::traits::pgen::HasMatchFields) { + auto match_fields = pgen.MatchFields(time); + CallMatchFields( + domain.fields.em, + domain.mesh.flds_bc(), + match_fields, + domain.mesh.metric, + xg_edge, + ds, + tags, + range_min, + range_max); + } else if constexpr (arch::traits::pgen::HasMatchFieldsInX2) { + auto match_fields = pgen.MatchFieldsInX2(time); + CallMatchFields( + domain.fields.em, + domain.mesh.flds_bc(), + match_fields, + domain.mesh.metric, + xg_edge, + ds, + tags, + range_min, + range_max); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + if constexpr (arch::traits::pgen::HasMatchFields) { + auto match_fields = pgen.MatchFields(time); + CallMatchFields( + domain.fields.em, + domain.mesh.flds_bc(), + match_fields, + domain.mesh.metric, + xg_edge, + ds, + tags, + range_min, + range_max); + } else if constexpr (arch::traits::pgen::HasMatchFieldsInX3) { + auto match_fields = pgen.MatchFieldsInX3(time); + CallMatchFields( + domain.fields.em, + domain.mesh.flds_bc(), + match_fields, + domain.mesh.metric, + xg_edge, + ds, + tags, + range_min, + range_max); + } + } + } else { + raise::Error("Invalid dimension", HERE); + } + } + + template + requires metric::traits::HasD + void AxisFieldsIn(dir::direction_t direction, + Domain& domain, + BCTags tags) { + /** + * axis boundaries + */ + if constexpr (M::CoordType != Coord::Cart) { + raise::ErrorIf(direction.get_dim() != in::x2, + "Invalid axis direction, should be x2", + HERE); + const auto i2_min = domain.mesh.i_min(in::x2); + const auto i2_max = domain.mesh.i_max(in::x2); + if (direction.get_sign() < 0) { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_min, + tags)); + } else { + Kokkos::parallel_for( + "AxisBCFields", + domain.mesh.n_all(in::x1), + kernel::bc::AxisBoundaries_kernel(domain.fields.em, + i2_max, + tags)); + } + } else { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Invalid coordinate type for axis BCs", HERE); + } + } + + template + requires metric::traits::HasD and metric::traits::HasCoordType and + metric::traits::HasTransform_i + void FixedFieldsIn(dir::direction_t direction, + Domain& domain, + const PG& pgen, + const prm::Parameters& engine_params, + BCTags tags) { + if constexpr (arch::traits::pgen::HasFixFieldsConst) { + const auto time = engine_params.get("time"); + /** + * fixed field boundaries + */ + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + raise::ErrorIf(dim != in::x1 and M::CoordType != Coord::Cart, + "Fixed BCs only implemented for x1 in " + "non-cartesian coordinates", + HERE); + std::vector xi_min, xi_max; + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + for (dim_t d { 0u }; d < M::Dim; ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { + if (sign > 0) { // + direction + xi_min.push_back(domain.mesh.n_all(dd) - N_GHOSTS); + xi_max.push_back(domain.mesh.n_all(dd)); + } else { // - direction + xi_min.push_back(0); + xi_max.push_back(N_GHOSTS); + } + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + std::vector comps; + if (tags & BC::E) { + comps.push_back(em::ex1); + comps.push_back(em::ex2); + comps.push_back(em::ex3); + } + if (tags & BC::B) { + comps.push_back(em::bx1); + comps.push_back(em::bx2); + comps.push_back(em::bx3); + } + raise::ErrorIf(M::CoordType != Coord::Cart and dim != in::x1, + "FixedFields cannot be used for non-cartesian metric", + HERE); + for (const auto& comp : comps) { + auto value = ZERO; + bool shouldset = false; + // if fix field function present, read from it + const auto newset = pgen.FixFieldsConst(time, + (bc_in)(sign * ((short)dim + 1)), + (em)comp); + value = newset.first; + shouldset = newset.second; + if (shouldset) { + // convert tetrad basis field (T) to contravariant (U) + real_t value_U = ZERO; + if (comp == em::ex1 or comp == em::bx1) { + value_U = domain.mesh.metric.template transform<1, Idx::T, Idx::U>( + { ZERO }, + value); + } else if (comp == em::ex2 or comp == em::bx2) { + value_U = domain.mesh.metric.template transform<2, Idx::T, Idx::U>( + { ZERO }, + value); + } else if (comp == em::ex3 or comp == em::bx3) { + value_U = domain.mesh.metric.template transform<3, Idx::T, Idx::U>( + { ZERO }, + value); + } else { + raise::Error("Invalid EM component", HERE); + } + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(xi_min[0], xi_max[0]), + comp), + value_U); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(xi_min[0], xi_max[0]), + std::make_pair(xi_min[1], xi_max[1]), + comp), + value_U); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.em, + std::make_pair(xi_min[0], xi_max[0]), + std::make_pair(xi_min[1], xi_max[1]), + std::make_pair(xi_min[2], xi_max[2]), + comp), + value_U); + } else { + raise::Error("Invalid dimension", HERE); + } + } + } + } else { + (void)direction; + (void)domain; + (void)tags; + } + } + + template + requires metric::traits::HasD + void PerfectConductorFieldsIn(dir::direction_t direction, + Domain& domain, + BCTags tags) { + /** + * perfect conductor field boundaries + */ + if constexpr (M::CoordType != Coord::Cart) { + (void)direction; + (void)domain; + (void)tags; + raise::Error( + "Perfect conductor BCs only applicable to cartesian coordinates", + HERE); + } else { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + + std::vector xi_min, xi_max; + + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + + for (auto d { 0u }; d < M::Dim; ++d) { + const auto dd = all_dirs[d]; + if (dim == dd) { + xi_min.push_back(0); + xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } + } + raise::ErrorIf(xi_min.size() != xi_max.size() or + xi_min.size() != static_cast(M::Dim), + "Invalid range size", + HERE); + + range_t range; + if constexpr (M::Dim == Dim::_1D) { + range = CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (M::Dim == Dim::_2D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else if constexpr (M::Dim == Dim::_3D) { + range = CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); + } else { + raise::Error("Invalid dimension", HERE); + } + std::size_t i_edge; + if (sign > 0) { + i_edge = domain.mesh.i_max(dim); + } else { + i_edge = domain.mesh.i_min(dim); + } + + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + i_edge, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + i_edge, + tags)); + } + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + i_edge, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + i_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else { + if constexpr (M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + i_edge, + tags)); + } else { + Kokkos::parallel_for( + "ConductorFields", + range, + kernel::bc::ConductorBoundaries_kernel( + domain.fields.em, + i_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } + } + } + + template + void AtmosphereFieldsIn(dir::direction_t direction, + Domain& domain, + const M& global_metric, + const Grid& global_grid, + const PG& pgen, + const SimulationParams& params, + const prm::Parameters& engine_params, + BCTags tags) { + /** + * atmosphere field boundaries + */ + if constexpr (arch::traits::pgen::HasAtmFields) { + const auto time = engine_params.get("time"); + const auto [sign, dim, xg_min, xg_max] = GetAtmosphereExtent(direction, + global_metric, + global_grid, + params); + // get_atm_extent(direction); + const auto dd = static_cast(dim); + boundaries_t box; + boundaries_t incl_ghosts; + for (auto d { 0u }; d < M::Dim; ++d) { + if (d == dd) { + box.push_back({ xg_min, xg_max }); + if (sign > 0) { + incl_ghosts.push_back({ false, true }); + } else { + incl_ghosts.push_back({ true, false }); + } + } else { + box.push_back(Range::All); + incl_ghosts.push_back({ true, true }); + } + } + if (not domain.mesh.Intersects(box)) { + return; + } + const auto intersect_range = domain.mesh.ExtentToRange(box, incl_ghosts); + tuple_t range_min { 0 }; + tuple_t range_max { 0 }; + + for (auto d { 0u }; d < M::Dim; ++d) { + range_min[d] = intersect_range[d].first; + range_max[d] = intersect_range[d].second; + } + auto atm_fields = pgen.AtmFields(time); + std::size_t il_edge; + if (sign > 0) { + il_edge = range_min[dd] - N_GHOSTS; + } else { + il_edge = range_max[dd] - 1 - N_GHOSTS; + } + const auto range = CreateRangePolicy(range_min, range_max); + if (dim == in::x1) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + if (sign > 0) { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } else { + Kokkos::parallel_for( + "AtmosphereBCFields", + range, + kernel::bc::EnforcedBoundaries_kernel( + domain.fields.em, + atm_fields, + domain.mesh.metric, + il_edge, + tags)); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else { + raise::Error("Invalid dimension", HERE); + } + } else { + (void)direction; + (void)domain; + (void)tags; + } + } + + template + requires metric::traits::HasD + void CustomFieldsIn(dir::direction_t direction, + Domain& domain, + BCTags tags) { + (void)direction; + (void)domain; + (void)tags; + raise::Error("Custom boundaries not implemented", HERE); + } + + template + requires metric::traits::HasD + void FieldBoundaries(Domain& domain, + const M& global_metric, + const Grid& global_grid, + const PG& pgen, + const prm::Parameters& engine_params, + const SimulationParams& params, + BCTags tags) { + for (auto& direction : dir::Directions::orth) { + if (global_grid.flds_bc_in(direction) == FldsBC::MATCH) { + MatchFieldsIn(direction, + domain, + global_grid, + pgen, + engine_params, + params, + tags); + } else if (global_grid.flds_bc_in(direction) == FldsBC::AXIS) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::AXIS) { + AxisFieldsIn(direction, domain, tags); + } + } else if (global_grid.flds_bc_in(direction) == FldsBC::ATMOSPHERE) { + AtmosphereFieldsIn(direction, + domain, + global_metric, + global_grid, + pgen, + params, + engine_params, + tags); + } else if (global_grid.flds_bc_in(direction) == FldsBC::FIXED) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::FIXED) { + FixedFieldsIn(direction, domain, pgen, engine_params, tags); + } + } else if (global_grid.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + PerfectConductorFieldsIn(direction, domain, tags); + } + } else if (global_grid.flds_bc_in(direction) == FldsBC::CUSTOM) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CUSTOM) { + CustomFieldsIn(direction, domain, tags); + } + } else if (global_grid.flds_bc_in(direction) == FldsBC::HORIZON) { + raise::Error("HORIZON BCs only applicable for GR", HERE); + } + } // loop over directions + } + + } // namespace srpic +} // namespace ntt + +#endif // ENGINES_SRPIC_FIELDS_BCS_H diff --git a/src/engines/srpic/fieldsolvers.h b/src/engines/srpic/fieldsolvers.h new file mode 100644 index 000000000..bc68cee3a --- /dev/null +++ b/src/engines/srpic/fieldsolvers.h @@ -0,0 +1,192 @@ +#ifndef ENGINES_SRPIC_FIELDSOLVERS_H +#define ENGINES_SRPIC_FIELDSOLVERS_H + +#include "enums.h" +#include "global.h" + +#include "utils/log.h" +#include "utils/numeric.h" +#include "utils/param_container.h" + +#include "archetypes/traits.h" +#include "engines/srpic/utils.h" +#include "framework/domain/domain.h" +#include "framework/parameters/parameters.h" +#include "kernels/ampere_mink.hpp" +#include "kernels/ampere_sr.hpp" +#include "kernels/faraday_mink.hpp" +#include "kernels/faraday_sr.hpp" + +namespace ntt { + namespace srpic { + + template + void Faraday(Domain& domain, + const prm::Parameters& engine_params, + const SimulationParams& params, + real_t fraction = ONE) { + logger::Checkpoint("Launching Faraday kernel", HERE); + const auto dt = engine_params.get("dt"); + + const auto dT = fraction * + params.template get( + "algorithms.timestep.correction") * + dt; + if constexpr (M::CoordType == Coord::Cart) { + // minkowski case + const auto dx = math::sqrt(domain.mesh.metric.template h_<1, 1>({})); + const auto deltax = params.template get( + "algorithms.fieldsolver.delta_x"); + const auto deltay = params.template get( + "algorithms.fieldsolver.delta_y"); + const auto betaxy = params.template get( + "algorithms.fieldsolver.beta_xy"); + const auto betayx = params.template get( + "algorithms.fieldsolver.beta_yx"); + const auto deltaz = params.template get( + "algorithms.fieldsolver.delta_z"); + const auto betaxz = params.template get( + "algorithms.fieldsolver.beta_xz"); + const auto betazx = params.template get( + "algorithms.fieldsolver.beta_zx"); + const auto betayz = params.template get( + "algorithms.fieldsolver.beta_yz"); + const auto betazy = params.template get( + "algorithms.fieldsolver.beta_zy"); + real_t coeff1, coeff2; + if constexpr (M::Dim == Dim::_2D) { + coeff1 = dT / SQR(dx); + coeff2 = dT; + } else { + coeff1 = dT / dx; + coeff2 = ZERO; + } + Kokkos::parallel_for("Faraday", + domain.mesh.rangeActiveCells(), + kernel::mink::Faraday_kernel(domain.fields.em, + coeff1, + coeff2, + deltax, + deltay, + betaxy, + betayx, + deltaz, + betaxz, + betazx, + betayz, + betazy)); + } else { + Kokkos::parallel_for("Faraday", + domain.mesh.rangeActiveCells(), + kernel::sr::Faraday_kernel(domain.fields.em, + domain.mesh.metric, + dT, + domain.mesh.flds_bc())); + } + } + + template + void Ampere(Domain& domain, + const prm::Parameters& engine_params, + const SimulationParams& params, + real_t fraction = ONE) { + logger::Checkpoint("Launching Ampere kernel", HERE); + const auto dt = engine_params.get("dt"); + + const auto dT = fraction * + params.template get( + "algorithms.timestep.correction") * + dt; + auto range = RangeWithAxisBCs(domain); + if constexpr (M::CoordType == Coord::Cart) { + // minkowski case + const auto dx = math::sqrt(domain.mesh.metric.template h_<1, 1>({})); + real_t coeff1, coeff2; + if constexpr (M::Dim == Dim::_2D) { + coeff1 = dT / SQR(dx); + coeff2 = dT; + } else { + coeff1 = dT / dx; + coeff2 = ZERO; + } + + Kokkos::parallel_for( + "Ampere", + range, + kernel::mink::Ampere_kernel(domain.fields.em, coeff1, coeff2)); + } else { + const auto ni2 = domain.mesh.n_active(in::x2); + Kokkos::parallel_for("Ampere", + range, + kernel::sr::Ampere_kernel(domain.fields.em, + domain.mesh.metric, + dT, + ni2, + domain.mesh.flds_bc())); + } + } + + template + void CurrentsAmpere(Domain& domain, + const prm::Parameters& engine_params, + const SimulationParams& params, + const PG& pgen) { + logger::Checkpoint("Launching Ampere kernel for adding currents", HERE); + const auto dt = engine_params.get("dt"); + + const auto q0 = params.template get("scales.q0"); + const auto n0 = params.template get("scales.n0"); + const auto B0 = params.template get("scales.B0"); + if constexpr (M::CoordType == Coord::Cart) { + // minkowski case + const auto V0 = params.template get("scales.V0"); + const auto ppc0 = params.template get("particles.ppc0"); + const auto coeff = -dt * q0 / (B0 * V0); + if constexpr (arch::traits::pgen::HasExtCurrent) { + const std::vector xmin { domain.mesh.extent(in::x1).first, + domain.mesh.extent(in::x2).first, + domain.mesh.extent(in::x3).first }; + const auto ext_current = pgen.ext_current; + const auto dx = domain.mesh.metric.template sqrt_h_<1, 1>({}); + Kokkos::parallel_for( + "Ampere", + domain.mesh.rangeActiveCells(), + kernel::mink::CurrentsAmpere_kernel( + domain.fields.em, + domain.fields.cur, + coeff, + ppc0, + ext_current, + xmin, + dx)); + } else { + Kokkos::parallel_for( + "Ampere", + domain.mesh.rangeActiveCells(), + kernel::mink::CurrentsAmpere_kernel(domain.fields.em, + domain.fields.cur, + coeff, + ppc0)); + } + } else { + // non-minkowski + const auto coeff = -dt * q0 * n0 / B0; + auto range = RangeWithAxisBCs(domain); + const auto ni2 = domain.mesh.n_active(in::x2); + Kokkos::parallel_for( + "Ampere", + range, + kernel::sr::CurrentsAmpere_kernel(domain.fields.em, + domain.fields.cur, + domain.mesh.metric, + coeff, + ONE / n0, + ni2, + domain.mesh.flds_bc())); + } + } + + } // namespace srpic +} // namespace ntt + +#endif // ENGINES_SRPIC_FIELDSOLVERS_H diff --git a/src/engines/srpic/particle_pusher.h b/src/engines/srpic/particle_pusher.h new file mode 100644 index 000000000..47189e98a --- /dev/null +++ b/src/engines/srpic/particle_pusher.h @@ -0,0 +1,382 @@ +#ifndef ENGINES_SRPIC_PARTICLE_PUSHER_H +#define ENGINES_SRPIC_PARTICLE_PUSHER_H + +#include "enums.h" +#include "global.h" + +#include "utils/comparators.h" +#include "utils/log.h" +#include "utils/numeric.h" +#include "utils/param_container.h" + +#include "metrics/traits.h" + +#include "archetypes/traits.h" +#include "engines/srpic/utils.h" +#include "framework/domain/domain.h" +#include "framework/domain/grid.h" +#include "framework/parameters/parameters.h" +#include "kernels/emission/compton.hpp" +#include "kernels/emission/emission.hpp" +#include "kernels/emission/synchrotron.hpp" +#include "kernels/emission/traits.h" +#include "kernels/particle_pusher_sr.hpp" + +namespace ntt { + namespace srpic { + + template + void CallPusher_WithExternalFieldFlag( + Domain& domain, + const SimulationParams& params, + const kernel::sr::PusherParams& pusher_params, + kernel::sr::PusherArrays& pusher_arrays, + EmissionTypeFlag emission_policy_flag, + const range_t& range, + const ndfield_t& EB, + const M& metric, + const PG& pgen, + const F& external_fields) { + if (emission_policy_flag == EmissionType::NONE) { + const auto no_emission = kernel::NoEmissionPolicy_t {}; + Kokkos::parallel_for( + "ParticlePusher", + range, + kernel::sr::Pusher_kernel( + pusher_params, + pusher_arrays, + EB, + metric, + external_fields, + no_emission)); + } else if (emission_policy_flag == EmissionType::SYNCHROTRON) { + const auto photon_species = params.get( + "radiation.emission.synchrotron.photon_species"); + raise::ErrorIf(photon_species > domain.species.size(), + "Invalid photon_species for Synchrotron emission", + HERE); + auto& emitted_species = domain.species[photon_species - 1]; + raise::ErrorIf(not cmp::AlmostZero_host(emitted_species.mass()), + "Emitted photon species must have zero mass", + HERE); + raise::ErrorIf(not cmp::AlmostZero_host(emitted_species.charge()), + "Emitted photon species must have zero charge", + HERE); + const auto emission_policy = kernel::emission::Synchrotron( + emitted_species, + photon_species, + pusher_params.mass, + pusher_params.charge, + pusher_params.radiative_drag_flags, + domain.index(), + params, + domain.random_pool()); + static_assert( + kernel::traits::emission::IsValid, M>, + "Synchrotron emission policy does not satisfy the required " + "interface"); + Kokkos::parallel_for( + "ParticlePusher", + range, + kernel::sr::Pusher_kernel( + pusher_params, + pusher_arrays, + EB, + metric, + external_fields, + emission_policy)); + const auto n_inj = emission_policy.numbers_injected(); + raise::ErrorIf(n_inj.size() != 1, + "Synchrotron emission should only inject one species", + HERE); + domain.species[photon_species - 1].set_npart( + emitted_species.npart() + n_inj[0]); + domain.species[photon_species - 1].set_counter( + emitted_species.counter() + n_inj[0]); + } else if (emission_policy_flag == EmissionType::COMPTON) { + const auto photon_species = params.get( + "radiation.emission.compton.photon_species"); + raise::ErrorIf(photon_species > domain.species.size(), + "Invalid photon_species for Compton emission", + HERE); + auto& emitted_species = domain.species[photon_species - 1]; + raise::ErrorIf(not cmp::AlmostZero_host(emitted_species.mass()), + "Emitted photon species must have zero mass", + HERE); + raise::ErrorIf(not cmp::AlmostZero_host(emitted_species.charge()), + "Emitted photon species must have zero charge", + HERE); + const auto emission_policy = kernel::emission::Compton( + emitted_species, + photon_species, + pusher_params.mass, + pusher_params.charge, + pusher_params.radiative_drag_flags, + domain.index(), + params, + domain.random_pool()); + static_assert( + kernel::traits::emission::IsValid, M>, + "Compton emission policy does not satisfy the required interface"); + Kokkos::parallel_for( + "ParticlePusher", + range, + kernel::sr::Pusher_kernel( + pusher_params, + pusher_arrays, + EB, + metric, + external_fields, + emission_policy)); + const auto n_inj = emission_policy.numbers_injected(); + raise::ErrorIf(n_inj.size() != 1, + "Compton emission should only inject one species", + HERE); + domain.species[photon_species - 1].set_npart( + emitted_species.npart() + n_inj[0]); + domain.species[photon_species - 1].set_counter( + emitted_species.counter() + n_inj[0]); + } else if (emission_policy_flag == EmissionType::CUSTOM) { + if constexpr (arch::traits::pgen::HasEmissionPolicy) { + const auto emission_policy = pgen.EmissionPolicy(pusher_params.time, + pusher_params.species_index, + domain); + static_assert( + kernel::traits::emission::IsValid, + "Custom emission policy does not satisfy the required " + "interface"); + Kokkos::parallel_for( + "ParticlePusher", + range, + kernel::sr::Pusher_kernel( + pusher_params, + pusher_arrays, + EB, + metric, + external_fields, + emission_policy)); + const auto emitted_species = emission_policy.emitted_species_indices(); + const auto n_inj = emission_policy.numbers_injected(); + raise::ErrorIf(emitted_species.size() != n_inj.size(), + "Emission policy emitted_species_indices and " + "numbers_injected must have the same size", + HERE); + for (auto i = 0u; i < emitted_species.size(); ++i) { + const auto sp_idx = emitted_species[i]; + raise::ErrorIf(sp_idx > domain.species.size(), + "Invalid emitted species index from custom " + "emission policy", + HERE); + domain.species[sp_idx - 1].set_npart( + domain.species[sp_idx - 1].npart() + n_inj[i]); + domain.species[sp_idx - 1].set_counter( + domain.species[sp_idx - 1].counter() + n_inj[i]); + } + } else { + raise::Error("Custom emission policy flag is set but problem " + "generator does not define an emission policy", + HERE); + } + } else { + raise::Error("Unrecognized emission policy flag", HERE); + } + } + + template + void CallPusher_WithAtmFlag(Domain& domain, + const SimulationParams& params, + const kernel::sr::PusherParams& pusher_params, + kernel::sr::PusherArrays& pusher_arrays, + EmissionTypeFlag emission_policy_flag, + const range_t& range, + const ndfield_t& EB, + const M& metric, + const PG& pgen) { + if constexpr (arch::traits::pgen::HasExternalFields) { + auto [apply_extfields, + external_fields] = pgen.ExternalFields(pusher_params.time, + pusher_params.species_index, + domain); + if (apply_extfields) { + CallPusher_WithExternalFieldFlag( + domain, + params, + pusher_params, + pusher_arrays, + emission_policy_flag, + range, + domain.fields.em, + domain.mesh.metric, + pgen, + external_fields); + return; + } + } + CallPusher_WithExternalFieldFlag( + domain, + params, + pusher_params, + pusher_arrays, + emission_policy_flag, + range, + domain.fields.em, + domain.mesh.metric, + pgen, + kernel::sr::NoField_t {}); + } + + template + void CallPusher(Domain& domain, + const SimulationParams& params, + const kernel::sr::PusherParams& pusher_params, + kernel::sr::PusherArrays& pusher_arrays, + EmissionTypeFlag emission_policy_flag, + const range_t& range, + const ndfield_t& EB, + const M& metric, + const PG& pgen, + bool has_atmosphere) { + if (not has_atmosphere) { + CallPusher_WithAtmFlag(domain, + params, + pusher_params, + pusher_arrays, + emission_policy_flag, + range, + domain.fields.em, + domain.mesh.metric, + pgen); + } else { + CallPusher_WithAtmFlag(domain, + params, + pusher_params, + pusher_arrays, + emission_policy_flag, + range, + domain.fields.em, + domain.mesh.metric, + pgen); + } + } + + template + requires metric::traits::HasD + void ParticlePush(Domain& domain, + const Grid& global_grid, + const M& global_metric, + const prm::Parameters& engine_params, + const SimulationParams& params, + const PG& pgen) { + const auto dt = engine_params.get("dt"); + const auto time = engine_params.get("time"); + + real_t gx1 { ZERO }, gx2 { ZERO }, gx3 { ZERO }, ds { ZERO }; + real_t x_surf { ZERO }; + bool has_atmosphere = false; + for (auto& direction : dir::Directions::orth) { + if (global_grid.prtl_bc_in(direction) == PrtlBC::ATMOSPHERE) { + raise::ErrorIf(has_atmosphere, + "Only one direction is allowed to have atm boundaries", + HERE); + has_atmosphere = true; + const auto g = params.template get( + "grid.boundaries.atmosphere.g"); + ds = params.template get("grid.boundaries.atmosphere.ds"); + const auto [sign, dim, xg_min, xg_max] = + GetAtmosphereExtent(direction, global_metric, global_grid, params); + if (dim == in::x1) { + gx1 = sign > 0 ? g : -g; + gx2 = ZERO; + gx3 = ZERO; + } else if (dim == in::x2) { + gx1 = ZERO; + gx2 = sign > 0 ? g : -g; + gx3 = ZERO; + } else if (dim == in::x3) { + gx1 = ZERO; + gx2 = ZERO; + gx3 = sign > 0 ? g : -g; + } else { + raise::Error("Invalid dimension", HERE); + } + if (sign > 0) { + x_surf = xg_min; + } else { + x_surf = xg_max; + } + } + } + for (auto& species : domain.species) { + if ((species.pusher() == ParticlePusher::NONE) or (species.npart() == 0)) { + continue; + } + species.set_unsorted(); + logger::Checkpoint( + fmt::format("Launching particle pusher kernel for %d [%s] : %lu", + species.index(), + species.label().c_str(), + species.npart()), + HERE); + + kernel::sr::PusherParams pusher_params {}; + pusher_params.species_index = species.index(); + pusher_params.pusher_flags = species.pusher(); + pusher_params.radiative_drag_flags = species.radiative_drag_flags(); + pusher_params.mass = species.mass(); + pusher_params.charge = species.charge(); + pusher_params.time = time; + pusher_params.dt = dt; + pusher_params.omegaB0 = params.template get("scales.omegaB0"); + pusher_params.ni1 = domain.mesh.n_active(in::x1); + pusher_params.ni2 = domain.mesh.n_active(in::x2); + pusher_params.ni3 = domain.mesh.n_active(in::x3); + pusher_params.boundaries = domain.mesh.prtl_bc(); + + if (has_atmosphere) { + pusher_params.atmosphere_params.set("gx1", gx1); + pusher_params.atmosphere_params.set("gx2", gx2); + pusher_params.atmosphere_params.set("gx3", gx3); + pusher_params.atmosphere_params.set("x_surf", x_surf); + pusher_params.atmosphere_params.set("ds", ds); + } + + if (species.pusher() & ParticlePusher::GCA) { + pusher_params.gca_params.set( + "larmor_max", + params.template get("algorithms.gca.larmor_max")); + pusher_params.gca_params.set( + "e_ovr_b_max", + params.template get("algorithms.gca.e_ovr_b_max")); + } + + if (species.radiative_drag_flags() & RadiativeDrag::SYNCHROTRON) { + pusher_params.radiative_drag_params.set( + "synchrotron_gamma_rad", + params.template get( + "radiation.drag.synchrotron.gamma_rad")); + } + + if (species.radiative_drag_flags() & RadiativeDrag::COMPTON) { + pusher_params.radiative_drag_params.set( + "compton_gamma_rad", + params.template get("radiation.drag.compton.gamma_rad")); + } + auto pusher_arrays = species.PusherKernelArrays(); + + CallPusher(domain, + params, + pusher_params, + pusher_arrays, + species.emission_policy_flag(), + species.rangeActiveParticles(), + domain.fields.em, + domain.mesh.metric, + pgen, + has_atmosphere); + } + } + + } // namespace srpic +} // namespace ntt + +#endif // ENGINES_SRPIC_PARTICLE_PUSHER_H diff --git a/src/engines/srpic/particles_bcs.h b/src/engines/srpic/particles_bcs.h new file mode 100644 index 000000000..5d444385f --- /dev/null +++ b/src/engines/srpic/particles_bcs.h @@ -0,0 +1,306 @@ +#ifndef ENGINES_SRPIC_PARTICLES_BCS_H +#define ENGINES_SRPIC_PARTICLES_BCS_H + +#include "enums.h" +#include "global.h" + +#include "arch/directions.h" +#include "utils/numeric.h" + +#include "metrics/traits.h" + +#include "archetypes/energy_dist.h" +#include "archetypes/particle_injector.h" +#include "archetypes/spatial_dist.h" +#include "engines/srpic/utils.h" +#include "framework/domain/domain.h" +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" +#include "kernels/particle_moments.hpp" + +namespace ntt { + namespace srpic { + + template + requires metric::traits::HasD && metric::traits::HasCoordType + void AtmosphereParticlesIn(dir::direction_t direction, + Metadomain& metadomain, + Domain& domain, + const SimulationParams& params, + InjTags tags) { + const auto [sign, dim, xg_min, xg_max] = srpic::GetAtmosphereExtent( + direction, + metadomain.mesh().metric, + metadomain.mesh(), + params); + + const auto x_surf = sign > 0 ? xg_min : xg_max; + const auto ds = params.template get( + "grid.boundaries.atmosphere.ds"); + const auto temp = params.template get( + "grid.boundaries.atmosphere.temperature"); + const auto height = params.template get( + "grid.boundaries.atmosphere.height"); + const auto species = params.template get>( + "grid.boundaries.atmosphere.species"); + const auto nmax = params.template get( + "grid.boundaries.atmosphere.density"); + + Kokkos::deep_copy(domain.fields.bckp, ZERO); + auto scatter_bckp = Kokkos::Experimental::create_scatter_view( + domain.fields.bckp); + const auto use_weights = M::CoordType != Coord::Cart; + const auto ni2 = domain.mesh.n_active(in::x2); + const auto inv_n0 = ONE / params.template get("scales.n0"); + + // compute the density of the two species + if (tags & Inj::AssumeEmpty) { + if constexpr (M::Dim == Dim::_1D) { + Kokkos::deep_copy( + Kokkos::subview(domain.fields.bckp, Kokkos::ALL, std::make_pair(0, 1)), + ZERO); + } else if constexpr (M::Dim == Dim::_2D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } else if constexpr (M::Dim == Dim::_3D) { + Kokkos::deep_copy(Kokkos::subview(domain.fields.bckp, + Kokkos::ALL, + Kokkos::ALL, + Kokkos::ALL, + std::make_pair(0, 1)), + ZERO); + } + } else { + for (const auto& sp : + std::vector { species.first, species.second }) { + auto& prtl_spec = domain.species[sp - 1]; + if (prtl_spec.npart() == 0) { + continue; + } + // clang-format off + Kokkos::parallel_for( + "ComputeMoments", + prtl_spec.rangeActiveParticles(), + kernel::ParticleMoments_kernel( + {}, scatter_bckp, 0, + prtl_spec.i1, prtl_spec.i2, prtl_spec.i3, + prtl_spec.dx1, prtl_spec.dx2, prtl_spec.dx3, + prtl_spec.ux1, prtl_spec.ux2, prtl_spec.ux3, + prtl_spec.phi, prtl_spec.weight, prtl_spec.tag, + prtl_spec.mass(), prtl_spec.charge(), + use_weights, + domain.mesh.metric, domain.mesh.flds_bc(), + ni2, inv_n0, 0)); + // clang-format on + prtl_spec.set_unsorted(); + } + Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); + metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); + } + + const auto maxwellian = arch::Maxwellian { + domain.mesh.metric, + domain.random_pool(), + temp + }; + + if (dim == in::x1) { + if (sign > 0) { + auto target_density = + arch::AtmosphereDensityProfile { + nmax, + height, + x_surf, + ds + }; + const auto spatial_dist = + arch::Replenish { + domain.mesh.metric, + domain.fields.bckp, + 0, + target_density, + nmax + }; + arch::InjectNonUniform( + params, + domain, + { species.first, species.second }, + { maxwellian, maxwellian }, + spatial_dist, + nmax, + use_weights); + } else { + auto target_density = + arch::AtmosphereDensityProfile { + nmax, + height, + x_surf, + ds + }; + const auto spatial_dist = + arch::Replenish { + domain.mesh.metric, + domain.fields.bckp, + 0, + target_density, + nmax + }; + arch::InjectNonUniform( + params, + domain, + { species.first, species.second }, + { maxwellian, maxwellian }, + spatial_dist, + nmax, + use_weights); + } + } else if (dim == in::x2) { + if (sign > 0) { + auto target_density = + arch::AtmosphereDensityProfile { + nmax, + height, + x_surf, + ds + }; + const auto spatial_dist = + arch::Replenish { + domain.mesh.metric, + domain.fields.bckp, + 0, + target_density, + nmax + }; + arch::InjectNonUniform( + params, + domain, + { species.first, species.second }, + { maxwellian, maxwellian }, + spatial_dist, + nmax, + use_weights); + } else { + auto target_density = + arch::AtmosphereDensityProfile { + nmax, + height, + x_surf, + ds + }; + const auto spatial_dist = + arch::Replenish { + domain.mesh.metric, + domain.fields.bckp, + 0, + target_density, + nmax + }; + arch::InjectNonUniform( + params, + domain, + { species.first, species.second }, + { maxwellian, maxwellian }, + spatial_dist, + nmax, + use_weights); + } + } else if (dim == in::x3) { + if (sign > 0) { + auto target_density = + arch::AtmosphereDensityProfile { + nmax, + height, + x_surf, + ds + }; + const auto spatial_dist = + arch::Replenish { + domain.mesh.metric, + domain.fields.bckp, + 0, + target_density, + nmax + }; + arch::InjectNonUniform( + params, + domain, + { species.first, species.second }, + { maxwellian, maxwellian }, + spatial_dist, + nmax, + use_weights); + } else { + auto target_density = + arch::AtmosphereDensityProfile { + nmax, + height, + x_surf, + ds + }; + const auto spatial_dist = + arch::Replenish { + domain.mesh.metric, + domain.fields.bckp, + 0, + target_density, + nmax + }; + arch::InjectNonUniform( + params, + domain, + { species.first, species.second }, + { maxwellian, maxwellian }, + spatial_dist, + nmax, + use_weights); + } + } else { + raise::Error("Invalid dimension", HERE); + } + return; + } + + template + requires metric::traits::HasD + void ParticleInjector(Metadomain& metadomain, + Domain& domain, + const SimulationParams& params, + InjTags tags = Inj::None) { + for (auto& direction : dir::Directions::orth) { + if (metadomain.mesh().prtl_bc_in(direction) == PrtlBC::ATMOSPHERE) { + AtmosphereParticlesIn(direction, metadomain, domain, params, tags); + } + } + } + + } // namespace srpic +} // namespace ntt + +#endif diff --git a/src/engines/srpic/srpic.hpp b/src/engines/srpic/srpic.hpp new file mode 100644 index 000000000..246f7f056 --- /dev/null +++ b/src/engines/srpic/srpic.hpp @@ -0,0 +1,195 @@ +/** + * @file engines/srpic.hpp + * @brief Simulation engien class which specialized on SRPIC + * @implements + * - ntt::SRPICEngine<> : ntt::Engine<> + * @cpp: + * - srpic.cpp + * @namespaces: + * - ntt:: + * @macros: + */ + +#ifndef ENGINES_SRPIC_SRPIC_H +#define ENGINES_SRPIC_SRPIC_H + +#include "enums.h" +#include "global.h" + +#include "utils/numeric.h" +#include "utils/timer.h" + +#include "engines/srpic/currents.h" +#include "engines/srpic/fields_bcs.h" +#include "engines/srpic/fieldsolvers.h" +#include "engines/srpic/particle_pusher.h" +#include "engines/srpic/particles_bcs.h" +#include "engines/traits.h" +#include "framework/domain/domain.h" +#include "framework/parameters/parameters.h" + +#include "engines/engine.hpp" +#include "pgen.hpp" + +#include +#include +#include + +namespace ntt { + + template + requires traits::engine::IsCompatibleWithSRPICEngine + class SRPICEngine : public Engine { + + using base_t = Engine; + using pgen_t = user::PGen; + using domain_t = Domain; + // contents + using base_t::m_metadomain; + using base_t::m_params; + using base_t::m_pgen; + // methods + using base_t::init; + // variables + using base_t::dt; + using base_t::max_steps; + using base_t::runtime; + using base_t::step; + using base_t::time; + + public: + static constexpr auto S { SimEngine::SRPIC }; + + SRPICEngine(const SimulationParams& params) : base_t { params } {} + + ~SRPICEngine() = default; + + void step_forward(timer::Timers& timers, domain_t& dom) override { + const auto fieldsolver_enabled = m_params.template get( + "algorithms.fieldsolver.enable"); + const auto deposit_enabled = m_params.template get( + "algorithms.deposit.enable"); + + if (step == 0) { + // communicate fields and apply BCs on the first timestep + m_metadomain.CommunicateFields(dom, Comm::B | Comm::E); + srpic::FieldBoundaries(dom, + m_metadomain.mesh().metric, + m_metadomain.mesh(), + m_pgen, + this->engineParams(), + m_params, + BC::B | BC::E); + srpic::ParticleInjector(m_metadomain, dom, m_params); + } + + if (fieldsolver_enabled) { + timers.start("FieldSolver"); + srpic::Faraday(dom, this->engineParams(), m_params, HALF); + timers.stop("FieldSolver"); + + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::B); + timers.stop("Communications"); + + timers.start("FieldBoundaries"); + srpic::FieldBoundaries(dom, + m_metadomain.mesh().metric, + m_metadomain.mesh(), + m_pgen, + this->engineParams(), + m_params, + BC::B); + timers.stop("FieldBoundaries"); + Kokkos::fence(); + } + + { + timers.start("ParticlePusher"); + srpic::ParticlePush(dom, + m_metadomain.mesh(), + m_metadomain.mesh().metric, + this->engineParams(), + m_params, + m_pgen); + timers.stop("ParticlePusher"); + + if (deposit_enabled) { + timers.start("CurrentDeposit"); + srpic::CurrentsDeposit(dom, this->engineParams()); + timers.stop("CurrentDeposit"); + + timers.start("Communications"); + m_metadomain.SynchronizeFields(dom, Comm::J); + m_metadomain.CommunicateFields(dom, Comm::J); + timers.stop("Communications"); + + timers.start("CurrentFiltering"); + srpic::CurrentsFilter(m_metadomain, dom, m_params); + timers.stop("CurrentFiltering"); + } + + timers.start("Communications"); + m_metadomain.CommunicateParticles(dom); + timers.stop("Communications"); + } + + if (fieldsolver_enabled) { + timers.start("FieldSolver"); + srpic::Faraday(dom, this->engineParams(), m_params, HALF); + timers.stop("FieldSolver"); + + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::B); + timers.stop("Communications"); + + timers.start("FieldBoundaries"); + srpic::FieldBoundaries(dom, + m_metadomain.mesh().metric, + m_metadomain.mesh(), + m_pgen, + this->engineParams(), + m_params, + BC::B); + timers.stop("FieldBoundaries"); + + timers.start("FieldSolver"); + srpic::Ampere(dom, this->engineParams(), m_params, ONE); + timers.stop("FieldSolver"); + + if (deposit_enabled) { + timers.start("FieldSolver"); + srpic::CurrentsAmpere(dom, this->engineParams(), m_params, m_pgen); + timers.stop("FieldSolver"); + } + + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, Comm::E | Comm::J); + timers.stop("Communications"); + + timers.start("FieldBoundaries"); + srpic::FieldBoundaries(dom, + m_metadomain.mesh().metric, + m_metadomain.mesh(), + m_pgen, + this->engineParams(), + m_params, + BC::E); + timers.stop("FieldBoundaries"); + } + + { + timers.start("Injector"); + srpic::ParticleInjector(m_metadomain, dom, m_params); + timers.stop("Injector"); + } + + timers.start("ParticleSort"); + m_metadomain.SortParticles(time, step, m_params, dom); + timers.stop("ParticleSort"); + } + }; + +} // namespace ntt + +#endif // ENGINES_SRPIC_SRPIC_H diff --git a/src/engines/srpic/utils.h b/src/engines/srpic/utils.h new file mode 100644 index 000000000..9779e55e4 --- /dev/null +++ b/src/engines/srpic/utils.h @@ -0,0 +1,127 @@ +#ifndef ENGINES_SRPIC_UTILS_H +#define ENGINES_SRPIC_UTILS_H + +#include "enums.h" +#include "global.h" + +#include "arch/directions.h" +#include "utils/numeric.h" + +#include "metrics/traits.h" + +#include "framework/domain/domain.h" +#include "framework/domain/grid.h" +#include "framework/parameters/parameters.h" + +#include + +namespace ntt { + namespace srpic { + + /** + * @brief Get the buffer region of the atmosphere and the direction + * @param direction direction in which the atmosphere is applied + * @return tuple: [sign of the direction, the direction (as in::), the min and max extent + * @note xg_min and xg_max are the extents where the fields are set, not the atmosphere itself + * @note i.e. + * + * fields set particles injected + * ghost zone | | + * v v v + * |....|...........|*******************..... -> x1 + * ^ ^ + * xg_min xg_max + * | | | + * |<-- buffer -->|<-- atmosphere -->| + * + * in this case the function returns { -1, in::x1, xg_min, xg_max } + */ + template + requires metric::traits::HasD && metric::traits::HasConvert + auto GetAtmosphereExtent(dir::direction_t direction, + const M& global_metric, + const Grid& global_grid, + const SimulationParams& params) + -> std::tuple { + const auto sign = direction.get_sign(); + const auto dim = direction.get_dim(); + const auto min_buff = params.template get( + "algorithms.current_filters") + + 2; + const auto buffer_ncells = min_buff > 5 ? min_buff : 5; + if (M::CoordType != Coord::Cart and (dim != in::x1 or sign > 0)) { + raise::Error("For non-cartesian coordinates atmosphere BCs is " + "possible only in -x1 (@ rmin)", + HERE); + } + real_t xg_min { ZERO }, xg_max { ZERO }; + ncells_t ig_min, ig_max; + if (sign > 0) { // + direction + ig_min = global_grid.n_active(dim) - buffer_ncells; + ig_max = global_grid.n_active(dim); + } else { // - direction + ig_min = 0; + ig_max = buffer_ncells; + } + + if (dim == in::x1) { + xg_min = global_metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = global_metric.template convert<1, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else if (dim == in::x2) { + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + xg_min = global_metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = global_metric.template convert<2, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else { + raise::Error("Invalid dimension", HERE); + } + } else if (dim == in::x3) { + if constexpr (M::Dim == Dim::_3D) { + xg_min = global_metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_min)); + xg_max = global_metric.template convert<3, Crd::Cd, Crd::Ph>( + static_cast(ig_max)); + } else { + raise::Error("Invalid dimension", HERE); + } + } else { + raise::Error("Invalid dimension", HERE); + } + return { sign, dim, xg_min, xg_max }; + } + + template + auto RangeWithAxisBCs(const Domain& domain) + -> range_t { + auto range = domain.mesh.rangeActiveCells(); + if constexpr (M::CoordType != Coord::Cart) { + /** + * @brief taking one extra cell in the x2 direction if AXIS BCs + */ + if constexpr (M::Dim == Dim::_2D) { + if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) { + range = CreateRangePolicy( + { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) }, + { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 }); + } + } else if constexpr (M::Dim == Dim::_3D) { + if (domain.mesh.flds_bc_in({ 0, +1, 0 }) == FldsBC::AXIS) { + range = CreateRangePolicy({ domain.mesh.i_min(in::x1), + domain.mesh.i_min(in::x2), + domain.mesh.i_min(in::x3) }, + { domain.mesh.i_max(in::x1), + domain.mesh.i_max(in::x2) + 1, + domain.mesh.i_max(in::x3) }); + } + } + } + return range; + } + + } // namespace srpic +} // namespace ntt + +#endif diff --git a/src/engines/traits.h b/src/engines/traits.h new file mode 100644 index 000000000..4050049b5 --- /dev/null +++ b/src/engines/traits.h @@ -0,0 +1,56 @@ +/** + * @file engine/traits.h + * @brief Defines a set of traits to check if an engine class satisfies certain + * conditions + * @implements + * - ntt::traits::engine::HasRun<> - checks if an engine has a run() method + * - ntt::traits::engine::IsCompatibleWithEngine<> - checks if a metric and + * pgen are compatible with a given simulation engine + * - ntt::traits::engine::IsCompatibleWithSRPICEngine<> - checks if a metric + * and pgen are compatible with the SRPIC engine + * - ntt::traits::engine::IsCompatibleWithGRPICEngine<> - checks if a metric + * and pgen are compatible with the GRPIC engine + * @namespaces: + * - ntt::traits::engine:: + */ +#ifndef ENGINES_TRAITS_H +#define ENGINES_TRAITS_H + +#include "metrics/traits.h" + +#include "archetypes/traits.h" + +#include + +namespace ntt { + namespace traits { + namespace engine { + + template class PG> + concept IsCompatibleWithEngine = + metric::traits::HasD and + arch::traits::pgen::check_compatibility::value(PG::engines) and + arch::traits::pgen::check_compatibility::value( + PG::metrics) and + arch::traits::pgen::check_compatibility::value(PG::dimensions); + + template class PG> + concept IsCompatibleWithSRPICEngine = + IsCompatibleWithEngine && + metric::traits::HasH_ij && metric::traits::HasConvert_i && + metric::traits::HasSqrtH_ij; + + template class PG> + concept IsCompatibleWithGRPICEngine = + IsCompatibleWithEngine; + + template + concept HasRun = requires(E& engine) { + { engine.run() } -> std::same_as; + }; + + } // namespace engine + } // namespace traits +} // namespace ntt + +#endif // ENGINES_TRAITS_H diff --git a/src/entity.cpp b/src/entity.cpp index 8d88e2654..9afb37cb2 100644 --- a/src/entity.cpp +++ b/src/entity.cpp @@ -3,19 +3,39 @@ #include "arch/traits.h" #include "utils/error.h" -#include "engines/engine_traits.h" +#include "archetypes/traits.h" #include "framework/simulation.h" #include "framework/specialization_registry.h" +#include "engines/grpic.hpp" +#include "engines/srpic/srpic.hpp" #include "pgen.hpp" - + #include +namespace ntt { + template + struct EngineSelector; + + template <> + struct EngineSelector { + template + using type = SRPICEngine; + }; + + template <> + struct EngineSelector { + template + using type = GRPICEngine; + }; +} // namespace ntt + template class M, Dimension D> static constexpr bool should_compile { - traits::check_compatibility::value(user::PGen>::engines) && - traits::check_compatibility::MetricType>::value(user::PGen>::metrics) && - traits::check_compatibility::value(user::PGen>::dimensions) + arch::traits::pgen::check_compatibility::value(user::PGen>::engines) && + arch::traits::pgen::check_compatibility::MetricType>::value( + user::PGen>::metrics) && + arch::traits::pgen::check_compatibility::value(user::PGen>::dimensions) }; template class M, Dimension D> @@ -26,7 +46,7 @@ void dispatch_engine(ntt::Simulation& sim) { sim.run::template type, M, D>(); } else { static_assert( - traits::always_false>::value, + ::traits::always_false>::value, "Unsupported engine"); } } diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index 014967870..df2bf4c69 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -4,16 +4,25 @@ # # @sources: # -# * parameters.cpp +# * parameters/parameters.cpp +# * parameters/particles.cpp +# * parameters/grid.cpp +# * parameters/output.cpp +# * parameters/algorithms.cpp +# * parameters/extra.cpp # * simulation.cpp # * domain/grid.cpp # * domain/metadomain.cpp -# * domain/communications.cpp -# * domain/checkpoint.cpp +# * domain/metadomain_sort.cpp +# * domain/metadomain_comm.cpp +# * domain/metadomain_chckpt.cpp +# * domain/metadomain_stats.cpp +# * domain/metadomain_io.cpp # * containers/particles.cpp +# * containers/particles_comm.cpp +# * containers/particles_io.cpp +# * containers/particles_sort.cpp # * containers/fields.cpp -# * domain/stats.cpp -# * domain/output.cpp # # @includes: # @@ -36,17 +45,24 @@ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(SOURCES - ${SRC_DIR}/parameters.cpp ${SRC_DIR}/simulation.cpp + ${SRC_DIR}/parameters/parameters.cpp + ${SRC_DIR}/parameters/particles.cpp + ${SRC_DIR}/parameters/grid.cpp + ${SRC_DIR}/parameters/output.cpp + ${SRC_DIR}/parameters/algorithms.cpp + ${SRC_DIR}/parameters/extra.cpp ${SRC_DIR}/domain/grid.cpp ${SRC_DIR}/domain/metadomain.cpp - ${SRC_DIR}/domain/communications.cpp - ${SRC_DIR}/domain/stats.cpp + ${SRC_DIR}/domain/metadomain_comm.cpp + ${SRC_DIR}/domain/metadomain_sort.cpp + ${SRC_DIR}/domain/metadomain_stats.cpp ${SRC_DIR}/containers/particles.cpp + ${SRC_DIR}/containers/particles_sort.cpp ${SRC_DIR}/containers/fields.cpp) if(${output}) - list(APPEND SOURCES ${SRC_DIR}/domain/output.cpp) - list(APPEND SOURCES ${SRC_DIR}/domain/checkpoint.cpp) + list(APPEND SOURCES ${SRC_DIR}/domain/metadomain_io.cpp) + list(APPEND SOURCES ${SRC_DIR}/domain/metadomain_chckpt.cpp) list(APPEND SOURCES ${SRC_DIR}/containers/fields_io.cpp) list(APPEND SOURCES ${SRC_DIR}/containers/particles_io.cpp) endif() @@ -58,7 +74,9 @@ add_library(ntt_framework ${SOURCES}) set(libs ntt_global ntt_metrics ntt_kernels ntt_output) add_dependencies(ntt_framework ${libs}) target_link_libraries(ntt_framework PUBLIC ${libs}) -target_link_libraries(ntt_framework PRIVATE stdc++fs) +if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "AppleClang") + target_link_libraries(ntt_framework PRIVATE stdc++fs) +endif() target_include_directories( ntt_framework diff --git a/src/framework/containers/particles.cpp b/src/framework/containers/particles.cpp index e9e516221..7682718a3 100644 --- a/src/framework/containers/particles.cpp +++ b/src/framework/containers/particles.cpp @@ -12,32 +12,36 @@ #include #include -#include namespace ntt { + template - Particles::Particles(spidx_t index, - const std::string& label, - float m, - float ch, - npart_t maxnpart, - const PrtlPusher& pusher, - bool use_tracking, - bool use_gca, - const Cooling& cooling, - unsigned short npld_r, - unsigned short npld_i) - : ParticleSpecies(index, - label, - m, - ch, - maxnpart, - pusher, - use_tracking, - use_gca, - cooling, - npld_r, - npld_i) { + Particles::Particles(spidx_t index, + const std::string& label, + float m, + float ch, + npart_t maxnpart, + timestep_t clearing_interval, + timestep_t spatial_sorting_interval, + ParticlePusherFlags particle_pusher_flags, + bool use_tracking, + RadiativeDragFlags radiative_drag_flags, + EmissionTypeFlag emission_policy_flag, + unsigned short npld_r, + unsigned short npld_i) + : ParticleSpecies { index, + label, + m, + ch, + maxnpart, + clearing_interval, + spatial_sorting_interval, + particle_pusher_flags, + use_tracking, + radiative_drag_flags, + emission_policy_flag, + npld_r, + npld_i } { if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { i1 = array_t { label + "_i1", maxnpart }; @@ -81,168 +85,28 @@ namespace ntt { } template - auto Particles::NpartsPerTagAndOffsets() const - -> std::pair, array_t> { - auto this_tag = tag; - const auto num_tags = ntags(); - array_t npptag { "nparts_per_tag", ntags() }; - - // count # of particles per each tag - auto npptag_scat = Kokkos::Experimental::create_scatter_view(npptag); - Kokkos::parallel_for( - "NpartPerTag", - rangeActiveParticles(), - Lambda(index_t p) { - auto npptag_acc = npptag_scat.access(); - if (this_tag(p) < 0 || this_tag(p) >= static_cast(num_tags)) { - raise::KernelError(HERE, "Invalid tag value"); - } - npptag_acc(this_tag(p)) += 1; - }); - Kokkos::Experimental::contribute(npptag, npptag_scat); - - // copy the count to a vector on the host - auto npptag_h = Kokkos::create_mirror_view(npptag); - Kokkos::deep_copy(npptag_h, npptag); - std::vector npptag_vec(num_tags); - for (auto t { 0u }; t < num_tags; ++t) { - npptag_vec[t] = npptag_h(t); - } - - // count the offsets on the host and copy to device - array_t tag_offsets("tag_offsets", num_tags - 3); - auto tag_offsets_h = Kokkos::create_mirror_view(tag_offsets); - - tag_offsets_h(0) = npptag_vec[2]; // offset for tag = 3 - for (auto t { 1u }; t < num_tags - 3; ++t) { - tag_offsets_h(t) = npptag_vec[t + 2] + tag_offsets_h(t - 1); - } - Kokkos::deep_copy(tag_offsets, tag_offsets_h); - - return { npptag_vec, tag_offsets }; - } - - template - void RemoveDeadInArray(array_t& arr, const array_t& indices_alive) { - npart_t n_alive = indices_alive.extent(0); - auto buffer = Kokkos::View("buffer", n_alive); - Kokkos::parallel_for( - "PopulateBufferAlive", - n_alive, - Lambda(index_t p) { buffer(p) = arr(indices_alive(p)); }); - - Kokkos::deep_copy( - Kokkos::subview(arr, std::make_pair(static_cast(0), n_alive)), - buffer); - } - - template - void RemoveDeadInArray(array_t& arr, const array_t& indices_alive) { - npart_t n_alive = indices_alive.extent(0); - auto buffer = array_t { "buffer", n_alive, arr.extent(1) }; - Kokkos::parallel_for( - "PopulateBufferAlive", - CreateRangePolicy({ 0, 0 }, { n_alive, arr.extent(1) }), - Lambda(index_t p, index_t l) { buffer(p, l) = arr(indices_alive(p), l); }); - - Kokkos::deep_copy( - Kokkos::subview(arr, - std::make_pair(static_cast(0), n_alive), - Kokkos::ALL), - buffer); - } - - template - void Particles::RemoveDead() { - npart_t n_alive = 0, n_dead = 0; - auto& this_tag = tag; - - Kokkos::parallel_reduce( - "CountDeadAlive", - rangeActiveParticles(), - Lambda(index_t p, npart_t & nalive, npart_t & ndead) { - nalive += (this_tag(p) == ParticleTag::alive); - ndead += (this_tag(p) == ParticleTag::dead); - if (this_tag(p) != ParticleTag::alive and this_tag(p) != ParticleTag::dead) { - raise::KernelError(HERE, "wrong particle tag"); - } - }, - n_alive, - n_dead); - - array_t indices_alive { "indices_alive", n_alive }; - array_t alive_counter { "counter_alive", 1 }; - - Kokkos::parallel_for( - "AliveIndices", - rangeActiveParticles(), - Lambda(index_t p) { - if (this_tag(p) == ParticleTag::alive) { - const auto idx = Kokkos::atomic_fetch_add(&alive_counter(0), 1); - indices_alive(idx) = p; - } - }); - - { - auto alive_counter_h = Kokkos::create_mirror_view(alive_counter); - Kokkos::deep_copy(alive_counter_h, alive_counter); - raise::ErrorIf(alive_counter_h(0) != n_alive, - "error in finding alive particle indices", - HERE); - } - - if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { - RemoveDeadInArray(i1, indices_alive); - RemoveDeadInArray(i1_prev, indices_alive); - RemoveDeadInArray(dx1, indices_alive); - RemoveDeadInArray(dx1_prev, indices_alive); - } - - if constexpr (D == Dim::_2D or D == Dim::_3D) { - RemoveDeadInArray(i2, indices_alive); - RemoveDeadInArray(i2_prev, indices_alive); - RemoveDeadInArray(dx2, indices_alive); - RemoveDeadInArray(dx2_prev, indices_alive); - } - - if constexpr (D == Dim::_3D) { - RemoveDeadInArray(i3, indices_alive); - RemoveDeadInArray(i3_prev, indices_alive); - RemoveDeadInArray(dx3, indices_alive); - RemoveDeadInArray(dx3_prev, indices_alive); - } - - RemoveDeadInArray(ux1, indices_alive); - RemoveDeadInArray(ux2, indices_alive); - RemoveDeadInArray(ux3, indices_alive); - RemoveDeadInArray(weight, indices_alive); - - if constexpr (D == Dim::_2D && C != Coord::Cart) { - RemoveDeadInArray(phi, indices_alive); - } - - if (npld_r() > 0) { - RemoveDeadInArray(pld_r, indices_alive); - } - - if (npld_i() > 0) { - RemoveDeadInArray(pld_i, indices_alive); - } - - Kokkos::Experimental::fill( - "TagAliveParticles", - Kokkos::DefaultExecutionSpace(), - Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), - ParticleTag::alive); - - Kokkos::Experimental::fill( - "TagDeadParticles", - Kokkos::DefaultExecutionSpace(), - Kokkos::subview(this_tag, std::make_pair(n_alive, n_alive + n_dead)), - ParticleTag::dead); - - set_npart(n_alive); - m_is_sorted = true; + auto Particles::PusherKernelArrays() -> kernel::sr::PusherArrays { + kernel::sr::PusherArrays pusher_arrays {}; + pusher_arrays.sp = index(); + pusher_arrays.i1 = i1; + pusher_arrays.i2 = i2; + pusher_arrays.i3 = i3; + pusher_arrays.i1_prev = i1_prev; + pusher_arrays.i2_prev = i2_prev; + pusher_arrays.i3_prev = i3_prev; + pusher_arrays.dx1 = dx1; + pusher_arrays.dx2 = dx2; + pusher_arrays.dx3 = dx3; + pusher_arrays.dx1_prev = dx1_prev; + pusher_arrays.dx2_prev = dx2_prev; + pusher_arrays.dx3_prev = dx3_prev; + pusher_arrays.ux1 = ux1; + pusher_arrays.ux2 = ux2; + pusher_arrays.ux3 = ux3; + pusher_arrays.phi = phi; + pusher_arrays.weight = weight; + pusher_arrays.tag = tag; + return pusher_arrays; } template struct Particles; diff --git a/src/framework/containers/particles.h b/src/framework/containers/particles.h index 144ca611c..c3ee1c1b5 100644 --- a/src/framework/containers/particles.h +++ b/src/framework/containers/particles.h @@ -5,6 +5,9 @@ * - ntt::Particles<> : ntt::ParticleSpecies * @cpp: * - particles.cpp + * - particles_io.cpp + * - particles_comm.cpp + * - particles_sort.cpp * @macros: * - MPI_ENABLED */ @@ -21,6 +24,8 @@ #include "utils/formatting.h" #include "framework/containers/species.h" +#include "framework/domain/grid.h" +#include "kernels/particle_pusher_sr.hpp" #include @@ -83,24 +88,28 @@ namespace ntt { * @param m The mass of the species * @param ch The charge of the species * @param maxnpart The maximum number of allocated particles for the species - * @param pusher The pusher assigned for the species + * @param clearing_interval The interval for clearing the particles + * @param spatial_sorting_interval The interval for spatial sorting of the particles + * @param particle_pusher_flags The pusher(s) assigned for the species * @param use_tracking Use particle tracking for the species - * @param use_gca Use hybrid GCA pusher for the species - * @param cooling The cooling mechanism assigned for the species + * @param radiative_drag_flags The radiative drag mechanism(s) assigned for the species + * @param emission_policy_flag The emission policy assigned for the species * @param npld_r The number of real-valued payloads for the species * @param npld_i The number of integer-valued payloads for the species */ - Particles(spidx_t index, - const std::string& label, - float m, - float ch, - npart_t maxnpart, - const PrtlPusher& pusher, - bool use_gca, - bool use_tracking, - const Cooling& cooling, - unsigned short npld_r = 0, - unsigned short npld_i = 0); + Particles(spidx_t index, + const std::string& label, + float m, + float ch, + npart_t maxnpart, + timestep_t clearing_interval, + timestep_t spatial_sorting_interval, + ParticlePusherFlags particle_pusher_flags, + bool use_tracking, + RadiativeDragFlags radiative_drag_flags, + EmissionTypeFlag emission_policy_flag, + unsigned short npld_r, + unsigned short npld_i); /** * @brief Constructor for the particle container @@ -113,10 +122,12 @@ namespace ntt { spec.mass(), spec.charge(), spec.maxnpart(), + spec.clearing_interval(), + spec.spatial_sorting_interval(), spec.pusher(), spec.use_tracking(), - spec.use_gca(), - spec.cooling(), + spec.radiative_drag_flags(), + spec.emission_policy_flag(), spec.npld_r(), spec.npld_i()) {} @@ -249,11 +260,23 @@ namespace ntt { */ void RemoveDead(); + /** + * @brief Sort particles spatially by their cell indices + * @param grid The grid object to get the cell information for sorting + */ + void SortSpatially(const Grid&); + /** * @brief Copy particle data from device to host. */ void SyncHostDevice(); + /** + * @brief Get the arrays required for the particle pusher kernel + * @returns The struct of arrays for the particle pusher kernel + */ + auto PusherKernelArrays() -> kernel::sr::PusherArrays; + #if defined(MPI_ENABLED) /** * @brief Communicate particles across neighboring meshblocks diff --git a/src/framework/containers/particles_comm.cpp b/src/framework/containers/particles_comm.cpp index 4d6d67118..1cf17efef 100644 --- a/src/framework/containers/particles_comm.cpp +++ b/src/framework/containers/particles_comm.cpp @@ -10,7 +10,6 @@ #include "utils/log.h" #include "framework/containers/particles.h" - #include "kernels/comm.hpp" #include diff --git a/src/framework/containers/particles_io.cpp b/src/framework/containers/particles_io.cpp index 8efff59ac..fb72ccc5f 100644 --- a/src/framework/containers/particles_io.cpp +++ b/src/framework/containers/particles_io.cpp @@ -5,20 +5,12 @@ #include "utils/formatting.h" #include "utils/log.h" -#include "metrics/kerr_schild.h" -#include "metrics/kerr_schild_0.h" -#include "metrics/minkowski.h" -#include "metrics/qkerr_schild.h" -#include "metrics/qspherical.h" -#include "metrics/spherical.h" - #include "framework/containers/particles.h" #include "framework/specialization_registry.h" +#include "kernels/prtls_to_phys.hpp" #include "output/utils/readers.h" #include "output/utils/writers.h" -#include "kernels/prtls_to_phys.hpp" - #include #include @@ -130,7 +122,10 @@ namespace ntt { npart_t nout_offset = 0; npart_t nout_total = nout; -#if defined(MPI_ENABLED) +#if !defined(MPI_ENABLED) + (void)domains_total; + (void)domains_offset; +#else auto nout_total_vec = std::vector(domains_total); MPI_Allgather(&nout, 1, @@ -140,7 +135,7 @@ namespace ntt { mpi::get_type(), MPI_COMM_WORLD); nout_total = 0; - for (auto r = 0; r < domains_total; ++r) { + for (auto r = 0u; r < domains_total; ++r) { if (r < domains_offset) { nout_offset += nout_total_vec[r]; } @@ -403,9 +398,9 @@ namespace ntt { { adios2::UnknownDim }); if (npld_r() > 0) { io.DefineVariable(fmt::format("s%d_pld_r", index()), - { adios2::UnknownDim, npld_r() }, - { adios2::UnknownDim, 0 }, - { adios2::UnknownDim, npld_r() }); + { adios2::UnknownDim }, + { adios2::UnknownDim }, + { adios2::UnknownDim }); } if (npld_i() > 0) { io.DefineVariable(fmt::format("s%d_pld_i", index()), @@ -436,7 +431,9 @@ namespace ntt { domains_offset); set_npart(npart_read); -#if defined(MPI_ENABLED) +#if !defined(MPI_ENABLED) + (void)domains_total; +#else { const auto npart_send = npart(); std::vector glob_nparts(domains_total); @@ -623,7 +620,7 @@ namespace ntt { mpi::get_type(), MPI_COMM_WORLD); npart_total = 0u; - for (auto r = 0; r < domains_total; ++r) { + for (auto r = 0u; r < domains_total; ++r) { if (r < domains_offset) { npart_offset += glob_nparts[r]; } @@ -814,6 +811,8 @@ namespace ntt { PARTICLES_OUTPUT_DECLARE(Dim::_3D, Coord::Cart) PARTICLES_OUTPUT_DECLARE(Dim::_2D, Coord::Sph) PARTICLES_OUTPUT_DECLARE(Dim::_2D, Coord::Qsph) + PARTICLES_OUTPUT_DECLARE(Dim::_3D, Coord::Sph) + PARTICLES_OUTPUT_DECLARE(Dim::_3D, Coord::Qsph) #undef PARTICLES_OUTPUT_DECLARE #define PARTICLES_OUTPUT_WRITE(S, M, D) \ diff --git a/src/framework/containers/particles_sort.cpp b/src/framework/containers/particles_sort.cpp new file mode 100644 index 000000000..2fd0e6011 --- /dev/null +++ b/src/framework/containers/particles_sort.cpp @@ -0,0 +1,334 @@ +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/sorting.h" + +#include "framework/containers/particles.h" +#include "framework/domain/grid.h" + +#include +#include +#include + +#include +#include + +namespace ntt { + + template + auto Particles::NpartsPerTagAndOffsets() const + -> std::pair, array_t> { + auto this_tag = tag; + const auto num_tags = ntags(); + array_t npptag { "nparts_per_tag", ntags() }; + + // count # of particles per each tag + auto npptag_scat = Kokkos::Experimental::create_scatter_view(npptag); + Kokkos::parallel_for( + "NpartPerTag", + rangeActiveParticles(), + Lambda(index_t p) { + auto npptag_acc = npptag_scat.access(); + if (this_tag(p) < 0 || this_tag(p) >= static_cast(num_tags)) { + raise::KernelError(HERE, "Invalid tag value"); + } + npptag_acc(this_tag(p)) += 1; + }); + Kokkos::Experimental::contribute(npptag, npptag_scat); + + // copy the count to a vector on the host + auto npptag_h = Kokkos::create_mirror_view(npptag); + Kokkos::deep_copy(npptag_h, npptag); + std::vector npptag_vec(num_tags); + for (auto t { 0u }; t < num_tags; ++t) { + npptag_vec[t] = npptag_h(t); + } + + // count the offsets on the host and copy to device + array_t tag_offsets("tag_offsets", num_tags - 3); + auto tag_offsets_h = Kokkos::create_mirror_view(tag_offsets); + + tag_offsets_h(0) = npptag_vec[2]; // offset for tag = 3 + for (auto t { 1u }; t < num_tags - 3; ++t) { + tag_offsets_h(t) = npptag_vec[t + 2] + tag_offsets_h(t - 1); + } + Kokkos::deep_copy(tag_offsets, tag_offsets_h); + + return { npptag_vec, tag_offsets }; + } + + template + void RemoveDeadInArray(array_t& arr, const array_t& indices_alive) { + npart_t n_alive = indices_alive.extent(0); + auto buffer = Kokkos::View("buffer", n_alive); + Kokkos::parallel_for( + "PopulateBufferAlive", + n_alive, + Lambda(index_t p) { buffer(p) = arr(indices_alive(p)); }); + + Kokkos::deep_copy( + Kokkos::subview(arr, std::make_pair(static_cast(0), n_alive)), + buffer); + } + + template + void RemoveDeadInArray(array_t& arr, const array_t& indices_alive) { + npart_t n_alive = indices_alive.extent(0); + auto buffer = array_t { "buffer", n_alive, arr.extent(1) }; + Kokkos::parallel_for( + "PopulateBufferAlive", + CreateRangePolicy({ 0, 0 }, { n_alive, arr.extent(1) }), + Lambda(index_t p, index_t l) { buffer(p, l) = arr(indices_alive(p), l); }); + + Kokkos::deep_copy( + Kokkos::subview(arr, + std::make_pair(static_cast(0), n_alive), + Kokkos::ALL), + buffer); + } + + template + void Particles::RemoveDead() { + npart_t n_alive = 0, n_dead = 0; + auto& this_tag = tag; + + Kokkos::parallel_reduce( + "CountDeadAlive", + rangeActiveParticles(), + Lambda(index_t p, npart_t & nalive, npart_t & ndead) { + nalive += (this_tag(p) == ParticleTag::alive); + ndead += (this_tag(p) == ParticleTag::dead); + if (this_tag(p) != ParticleTag::alive and this_tag(p) != ParticleTag::dead) { + raise::KernelError(HERE, "wrong particle tag"); + } + }, + n_alive, + n_dead); + + array_t indices_alive { "indices_alive", n_alive }; + array_t alive_counter { "counter_alive", 1 }; + + Kokkos::parallel_for( + "AliveIndices", + rangeActiveParticles(), + Lambda(index_t p) { + if (this_tag(p) == ParticleTag::alive) { + const auto idx = Kokkos::atomic_fetch_add(&alive_counter(0), 1); + indices_alive(idx) = p; + } + }); + + { + auto alive_counter_h = Kokkos::create_mirror_view(alive_counter); + Kokkos::deep_copy(alive_counter_h, alive_counter); + raise::ErrorIf(alive_counter_h(0) != n_alive, + "error in finding alive particle indices", + HERE); + } + + if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { + RemoveDeadInArray(i1, indices_alive); + RemoveDeadInArray(i1_prev, indices_alive); + RemoveDeadInArray(dx1, indices_alive); + RemoveDeadInArray(dx1_prev, indices_alive); + } + + if constexpr (D == Dim::_2D or D == Dim::_3D) { + RemoveDeadInArray(i2, indices_alive); + RemoveDeadInArray(i2_prev, indices_alive); + RemoveDeadInArray(dx2, indices_alive); + RemoveDeadInArray(dx2_prev, indices_alive); + } + + if constexpr (D == Dim::_3D) { + RemoveDeadInArray(i3, indices_alive); + RemoveDeadInArray(i3_prev, indices_alive); + RemoveDeadInArray(dx3, indices_alive); + RemoveDeadInArray(dx3_prev, indices_alive); + } + + RemoveDeadInArray(ux1, indices_alive); + RemoveDeadInArray(ux2, indices_alive); + RemoveDeadInArray(ux3, indices_alive); + RemoveDeadInArray(weight, indices_alive); + + if constexpr (D == Dim::_2D && C != Coord::Cart) { + RemoveDeadInArray(phi, indices_alive); + } + + if (npld_r() > 0) { + RemoveDeadInArray(pld_r, indices_alive); + } + + if (npld_i() > 0) { + RemoveDeadInArray(pld_i, indices_alive); + } + + Kokkos::Experimental::fill( + "TagAliveParticles", + Kokkos::DefaultExecutionSpace(), + Kokkos::subview(this_tag, std::make_pair(static_cast(0), n_alive)), + ParticleTag::alive); + + Kokkos::Experimental::fill( + "TagDeadParticles", + Kokkos::DefaultExecutionSpace(), + Kokkos::subview(this_tag, std::make_pair(n_alive, n_alive + n_dead)), + ParticleTag::dead); + + set_npart(n_alive); + m_is_sorted = true; + } + + template + void Particles::SortSpatially(const Grid& grid) { + const auto nx2 = grid.n_active(in::x2); + const auto nx3 = grid.n_active(in::x3); + const auto total_cells = grid.num_active(); + + array_t cell_indices { "cell_indices", npart() }; + + Kokkos::parallel_for( + "FillCellIndices", + rangeActiveParticles(), + sort::PositionToCellIndex { i1, i2, i3, tag, cell_indices, nx2, nx3, total_cells }); + const auto slice = range_tuple_t(0, npart()); + + using sorter_op_t = Kokkos::BinOp1D; + using sorter_t = Kokkos::BinSort; + auto bin_op = sorter_op_t { static_cast(total_cells + 1u), + 0u, + total_cells + 1u }; + auto sorter = sorter_t { cell_indices, bin_op, false }; + sorter.create_permute_vector(); + if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { + sorter.sort(Kokkos::subview(i1, slice)); + sorter.sort(Kokkos::subview(i1_prev, slice)); + sorter.sort(Kokkos::subview(dx1, slice)); + sorter.sort(Kokkos::subview(dx1_prev, slice)); + } + if constexpr (D == Dim::_2D or D == Dim::_3D) { + sorter.sort(Kokkos::subview(i2, slice)); + sorter.sort(Kokkos::subview(i2_prev, slice)); + sorter.sort(Kokkos::subview(dx2, slice)); + sorter.sort(Kokkos::subview(dx2_prev, slice)); + } + if constexpr (D == Dim::_3D) { + sorter.sort(Kokkos::subview(i3, slice)); + sorter.sort(Kokkos::subview(i3_prev, slice)); + sorter.sort(Kokkos::subview(dx3, slice)); + sorter.sort(Kokkos::subview(dx3_prev, slice)); + } + sorter.sort(Kokkos::subview(ux1, slice)); + sorter.sort(Kokkos::subview(ux2, slice)); + sorter.sort(Kokkos::subview(ux3, slice)); + sorter.sort(Kokkos::subview(weight, slice)); + sorter.sort(Kokkos::subview(tag, slice)); + if constexpr (D == Dim::_2D and C != Coord::Cart) { + sorter.sort(Kokkos::subview(phi, slice)); + } + for (auto pldr { 0u }; pldr < npld_r(); ++pldr) { + sorter.sort(Kokkos::subview(pld_r, slice, pldr)); + } + for (auto pldi { 0u }; pldi < npld_i(); ++pldi) { + sorter.sort(Kokkos::subview(pld_i, slice, pldi)); + } + } + +#define PARTICLES_SORT(D, C) \ + template auto Particles::NpartsPerTagAndOffsets() const \ + -> std::pair, array_t>; \ + template void Particles::RemoveDead(); \ + template void Particles::SortSpatially(const Grid&); + + PARTICLES_SORT(Dim::_1D, Coord::Cart) + PARTICLES_SORT(Dim::_2D, Coord::Cart) + PARTICLES_SORT(Dim::_3D, Coord::Cart) + PARTICLES_SORT(Dim::_2D, Coord::Sph) + PARTICLES_SORT(Dim::_2D, Coord::Qsph) + PARTICLES_SORT(Dim::_3D, Coord::Sph) + PARTICLES_SORT(Dim::_3D, Coord::Qsph) +#undef PARTICLES_SORT + +} // namespace ntt + +// template +// void AllocateArrayOnGrid(nddata_t& arr, +// const Grid& grid, +// const std::string& name) { +// if constexpr (D == Dim::_1D) { +// arr = nddata_t { name, grid.n_active(in::x1) }; +// } else if constexpr (D == Dim::_2D) { +// arr = nddata_t { name, grid.n_active(in::x1), grid.n_active(in::x2) }; +// } else if constexpr (D == Dim::_3D) { +// arr = nddata_t { name, +// grid.n_active(in::x1), +// grid.n_active(in::x2), +// grid.n_active(in::x3) }; +// } else { +// raise::Error("Unsupported dimension for array allocation", HERE); +// } +// } +// +// +// array_t cell_idx { "cell_indices", npart() }; +// const auto num_cells = grid.num_active(); +// +// nddata_t num_ppc; +// nddata_t disp_map; +// AllocateArrayOnGrid(num_ppc, grid, "num_ppc"); +// AllocateArrayOnGrid(disp_map, grid, "disp_map"); +// auto num_ppc_scatter = +// Kokkos::Experimental::create_scatter_view(num_ppc); Kokkos::parallel_for( +// "ComputeNumPPC", +// rangeActiveParticles(), +// Lambda(index_t p) { +// if (tag_p(p) != ParticleTag::alive) { +// return; +// } +// auto num_ppc_acc = num_ppc_scatter.access(); +// if constexpr (D == Dim::_1D) { +// num_ppc_acc(i1_p(p)) += 1u; +// } else if constexpr (D == Dim::_2D) { +// num_ppc_acc(i1_p(p), i2_p(p)) += 1u; +// } else { +// num_ppc_acc(i1_p(p), i2_p(p), i3_p(p)) += 1u; +// } +// }); +// Kokkos::Experimental::contribute(num_ppc, num_ppc_scatter); +// +// npart_t total_sum = 0u; +// Kokkos::parallel_scan( +// "ComputeDisplacementMap", +// total_cells, +// Lambda(index_t cell, npart_t & cumulative_sum, bool is_final) { +// ncells_t i1, i2, i3; +// if constexpr (D == Dim::_1D) { +// i1 = cell; +// } else if constexpr (D == Dim::_2D) { +// i1 = cell / nx2; +// i2 = cell % nx2; +// } else { +// i1 = cell / (nx2 * nx3); +// i2 = (cell % (nx2 * nx3)) / nx3; +// i3 = cell % nx3; +// } +// if (is_final) { +// if constexpr (D == Dim::_1D) { +// disp_map(i1) = cumulative_sum; +// } else if constexpr (D == Dim::_2D) { +// disp_map(i1, i2) = cumulative_sum; +// } else { +// disp_map(i1, i2, i3) = cumulative_sum; +// } +// } +// if constexpr (D == Dim::_1D) { +// cumulative_sum += num_ppc(i1); +// } else if constexpr (D == Dim::_2D) { +// cumulative_sum += num_ppc(i1, i2); +// } else { +// cumulative_sum += num_ppc(i1, i2, i3); +// } +// }, +// total_sum); diff --git a/src/framework/containers/species.h b/src/framework/containers/species.h index baf024874..5fcec5878 100644 --- a/src/framework/containers/species.h +++ b/src/framework/containers/species.h @@ -13,6 +13,9 @@ #include "enums.h" +#include "utils/formatting.h" +#include "utils/reporter.h" + #include namespace ntt { @@ -29,18 +32,22 @@ namespace ntt { const float m_charge; // Max number of allocated particles for the species npart_t m_maxnpart; + // Clearing interval for the species (0 means no clearing) + const timestep_t m_clearing_interval; + // Spatial sorting interval for the species (0 means no sorting) + const timestep_t m_spatial_sorting_interval; // Pusher assigned for the species - const PrtlPusher m_pusher; + const ParticlePusherFlags m_particle_pusher_flags; // Use particle tracking for the species const bool m_use_tracking; - // Use byrid gca pusher for the species - const bool m_use_gca; + // Radiative drag mechanism(s) assigned for the species + const RadiativeDragFlags m_radiative_drag_flags; - // Cooling drag mechanism assigned for the species - const Cooling m_cooling; + // Emission policy assigned for the species + const EmissionTypeFlag m_emission_policy_flag; // Number of payloads for the species const unsigned short m_npld_r; @@ -53,10 +60,12 @@ namespace ntt { , m_mass { 0.0 } , m_charge { 0.0 } , m_maxnpart { 0 } - , m_pusher { PrtlPusher::INVALID } + , m_clearing_interval { 0u } + , m_spatial_sorting_interval { 0u } + , m_particle_pusher_flags { ParticlePusher::NONE } , m_use_tracking { false } - , m_use_gca { false } - , m_cooling { Cooling::INVALID } + , m_radiative_drag_flags { RadiativeDrag::NONE } + , m_emission_policy_flag { EmissionType::NONE } , m_npld_r { 0 } , m_npld_i { 0 } {} @@ -68,33 +77,37 @@ namespace ntt { * @param m The mass of the species. * @param ch The charge of the species. * @param maxnpart The maximum number of allocated particles for the species. - * @param pusher The pusher assigned for the species. + * @param particle_pusher_flags The pusher(s) assigned for the species. * @param use_tracking Use particle tracking for the species. - * @param use_gca Use hybrid GCA pusher for the species. - * @param cooling The cooling mechanism assigned for the species. + * @param radiative_drag_flags The radiative drag mechanism(s) assigned for the species. + * @param emission_policy_flag The emission policy assigned for the species. * @param npld_r The number of real-valued payloads for the species * @param npld_i The number of integer-valued payloads for the species */ - ParticleSpecies(spidx_t index, - const std::string& label, - float m, - float ch, - npart_t maxnpart, - const PrtlPusher& pusher, - bool use_tracking, - bool use_gca, - const Cooling& cooling, - unsigned short npld_r = 0, - unsigned short npld_i = 0) + ParticleSpecies(spidx_t index, + const std::string& label, + float m, + float ch, + npart_t maxnpart, + timestep_t clearing_interval, + timestep_t spatial_sorting_interval, + ParticlePusherFlags particle_pusher_flags, + bool use_tracking, + RadiativeDragFlags radiative_drag_flags, + EmissionTypeFlag emission_policy_flag, + unsigned short npld_r, + unsigned short npld_i) : m_index { index } , m_label { std::move(label) } , m_mass { m } , m_charge { ch } , m_maxnpart { maxnpart } - , m_pusher { pusher } + , m_clearing_interval { clearing_interval } + , m_spatial_sorting_interval { spatial_sorting_interval } + , m_particle_pusher_flags { particle_pusher_flags } , m_use_tracking { use_tracking } - , m_use_gca { use_gca } - , m_cooling { cooling } + , m_radiative_drag_flags { radiative_drag_flags } + , m_emission_policy_flag { emission_policy_flag } , m_npld_r { npld_r } , m_npld_i { npld_i } { if (use_tracking) { @@ -144,8 +157,18 @@ namespace ntt { } [[nodiscard]] - auto pusher() const -> PrtlPusher { - return m_pusher; + auto clearing_interval() const -> timestep_t { + return m_clearing_interval; + } + + [[nodiscard]] + auto spatial_sorting_interval() const -> timestep_t { + return m_spatial_sorting_interval; + } + + [[nodiscard]] + auto pusher() const -> ParticlePusherFlags { + return m_particle_pusher_flags; } [[nodiscard]] @@ -154,13 +177,13 @@ namespace ntt { } [[nodiscard]] - auto use_gca() const -> bool { - return m_use_gca; + auto radiative_drag_flags() const -> RadiativeDragFlags { + return m_radiative_drag_flags; } [[nodiscard]] - auto cooling() const -> Cooling { - return m_cooling; + auto emission_policy_flag() const -> EmissionTypeFlag { + return m_emission_policy_flag; } [[nodiscard]] @@ -172,6 +195,50 @@ namespace ntt { auto npld_i() const -> unsigned short { return m_npld_i; } + + /* reporter -------------------------------------------------------------- */ + auto Report() const -> std::string { + std::string report = ""; + reporter::AddSubcategory(report, + 4, + fmt::format("Species #%d", index()).c_str()); + reporter::AddParam(report, 6, "Label", "%s", label().c_str()); + reporter::AddParam(report, 6, "Mass", "%.1f", mass()); + reporter::AddParam(report, 6, "Charge", "%.1f", charge()); + reporter::AddParam(report, 6, "Max #", "%d [per domain]", maxnpart()); + reporter::AddParam(report, + 6, + "Clearing interval", + "%s", + clearing_interval() == 0u + ? "OFF" + : fmt::format("%d", clearing_interval()).c_str()); + reporter::AddParam(report, + 6, + "Spatial sorting interval", + "%s", + spatial_sorting_interval() == 0u + ? "OFF" + : fmt::format("%d", spatial_sorting_interval()).c_str()); + reporter::AddParam(report, + 6, + "Pusher", + "%s", + ParticlePusher::to_string(pusher()).c_str()); + reporter::AddParam(report, + 6, + "Radiative drag", + "%s", + RadiativeDrag::to_string(radiative_drag_flags()).c_str()); + reporter::AddParam(report, + 6, + "Emission policy", + "%s", + EmissionType::to_string(emission_policy_flag()).c_str()); + reporter::AddParam(report, 6, "# of real-value payloads", "%d", npld_r()); + reporter::AddParam(report, 6, "# of integer-value payloads", "%d", npld_i()); + return report; + } }; } // namespace ntt diff --git a/src/framework/domain/domain.h b/src/framework/domain/domain.h index ca59b74e6..06ffd302d 100644 --- a/src/framework/domain/domain.h +++ b/src/framework/domain/domain.h @@ -45,6 +45,9 @@ #include "arch/directions.h" #include "utils/formatting.h" #include "utils/numeric.h" +#include "utils/reporter.h" + +#include "metrics/traits.h" #include "framework/containers/fields.h" #include "framework/containers/particles.h" @@ -58,9 +61,10 @@ #include namespace ntt { + template + requires metric::traits::HasD struct Domain { - static_assert(M::is_metric, "template arg for Mesh class has to be a metric"); static constexpr Dimension D { M::Dim }; Mesh mesh; @@ -154,10 +158,72 @@ namespace ntt { } /* setters -------------------------------------------------------------- */ - auto set_neighbor_idx(const dir::direction_t& dir, unsigned int idx) -> void { + auto set_neighbor_idx(const dir::direction_t& dir, unsigned int idx) + -> void { m_neighbor_idx[dir] = idx; } + /* printer overload ----------------------------------------------------- */ + auto Report() const -> std::string { + std::string report = ""; + reporter::AddSubcategory(report, + 4, + fmt::format("Domain #%d", index()).c_str()); +#if defined(MPI_ENABLED) + reporter::AddParam(report, 6, "Rank", "%d", mpi_rank()); +#endif + reporter::AddParam(report, + 6, + "Resolution", + "%s", + fmt::formatVector(mesh.n_active()).c_str()); + reporter::AddParam(report, + 6, + "Extent", + "%s", + fmt::formatVector(mesh.extent()).c_str()); + reporter::AddSubcategory(report, 6, "Boundary conditions"); + + reporter::AddLabel( + report, + 8 + 2 + 2 * M::Dim, + fmt::format("%-10s %-10s %-10s", "[flds]", "[prtl]", "[neighbor]").c_str()); + for (auto& direction : dir::Directions::all) { + const auto flds_bc = mesh.flds_bc_in(direction); + const auto prtl_bc = mesh.prtl_bc_in(direction); + bool has_sync = false; + auto neighbor_idx = neighbor_idx_in(direction); + if (flds_bc == FldsBC::SYNC || prtl_bc == PrtlBC::SYNC) { + has_sync = true; + } + reporter::AddUnlabeledParam( + report, + 8, + direction.to_string().c_str(), + "%-10s %-10s %-10s", + flds_bc.to_string(), + prtl_bc.to_string(), + has_sync ? std::to_string(neighbor_idx).c_str() : "."); + } + reporter::AddSubcategory(report, 6, "Memory footprint"); + auto flds_footprint = fields.memory_footprint(); + auto [flds_size, flds_unit] = reporter::Bytes2HumanReadable(flds_footprint); + reporter::AddParam(report, 8, "Fields", "%.2f %s", flds_size, flds_unit.c_str()); + if (species.size() > 0) { + reporter::AddSubcategory(report, 8, "Particles"); + } + for (auto& species : species) { + const auto str = fmt::format("Species #%d (%s)", + species.index(), + species.label().c_str()); + auto [size, + unit] = reporter::Bytes2HumanReadable(species.memory_footprint()); + reporter::AddParam(report, 10, str.c_str(), "%.2f %s", size, unit.c_str()); + } + report.pop_back(); + return report; + } + private: // index of the domain in the metadomain unsigned int m_index; @@ -175,8 +241,8 @@ namespace ntt { }; template - inline auto operator<<(std::ostream& os, - const Domain& domain) -> std::ostream& { + inline auto operator<<(std::ostream& os, const Domain& domain) + -> std::ostream& { os << "Domain #" << domain.index(); #if defined(MPI_ENABLED) os << " [MPI rank: " << domain.mpi_rank() << "]"; diff --git a/src/framework/domain/grid.cpp b/src/framework/domain/grid.cpp index fa409263b..1e223cbc5 100644 --- a/src/framework/domain/grid.cpp +++ b/src/framework/domain/grid.cpp @@ -85,8 +85,8 @@ namespace ntt { } template - auto Grid::rangeCellsOnHost( - const box_region_t& region) const -> range_h_t { + auto Grid::rangeCellsOnHost(const box_region_t& region) const + -> range_h_t { tuple_t imin, imax; for (auto i { 0u }; i < D; i++) { switch (region[i]) { @@ -162,8 +162,8 @@ namespace ntt { } template - auto Grid::rangeCells( - const tuple_t, D>& ranges) const -> range_t { + auto Grid::rangeCells(const tuple_t, D>& ranges) const + -> range_t { tuple_t imin, imax; for (auto i { 0u }; i < D; i++) { raise::ErrorIf((ranges[i][0] < -(int)N_GHOSTS) || @@ -177,6 +177,54 @@ namespace ntt { return CreateRangePolicy(imin, imax); } + template <> + auto Grid::flds_bc() const -> boundaries_t { + return { + { flds_bc_in({ -1 }), flds_bc_in({ 1 }) } + }; + } + + template <> + auto Grid::flds_bc() const -> boundaries_t { + return { + { flds_bc_in({ -1, 0 }), flds_bc_in({ 1, 0 }) }, + { flds_bc_in({ 0, -1 }), flds_bc_in({ 0, 1 }) } + }; + } + + template <> + auto Grid::flds_bc() const -> boundaries_t { + return { + { flds_bc_in({ -1, 0, 0 }), flds_bc_in({ 1, 0, 0 }) }, + { flds_bc_in({ 0, -1, 0 }), flds_bc_in({ 0, 1, 0 }) }, + { flds_bc_in({ 0, 0, -1 }), flds_bc_in({ 0, 0, 1 }) } + }; + } + + template <> + auto Grid::prtl_bc() const -> boundaries_t { + return { + { prtl_bc_in({ -1 }), prtl_bc_in({ 1 }) } + }; + } + + template <> + auto Grid::prtl_bc() const -> boundaries_t { + return { + { prtl_bc_in({ -1, 0 }), prtl_bc_in({ 1, 0 }) }, + { prtl_bc_in({ 0, -1 }), prtl_bc_in({ 0, 1 }) } + }; + } + + template <> + auto Grid::prtl_bc() const -> boundaries_t { + return { + { prtl_bc_in({ -1, 0, 0 }), prtl_bc_in({ 1, 0, 0 }) }, + { prtl_bc_in({ 0, -1, 0 }), prtl_bc_in({ 0, 1, 0 }) }, + { prtl_bc_in({ 0, 0, -1 }), prtl_bc_in({ 0, 0, 1 }) } + }; + } + template struct Grid; template struct Grid; template struct Grid; diff --git a/src/framework/domain/grid.h b/src/framework/domain/grid.h index 87b21b1f5..6fbe2fae3 100644 --- a/src/framework/domain/grid.h +++ b/src/framework/domain/grid.h @@ -62,10 +62,13 @@ #ifndef FRAMEWORK_DOMAIN_GRID_H #define FRAMEWORK_DOMAIN_GRID_H +#include "enums.h" #include "global.h" +#include "arch/directions.h" #include "arch/kokkos_aliases.h" #include "utils/error.h" +#include "utils/numeric.h" #include @@ -73,10 +76,29 @@ namespace ntt { template struct Grid { - Grid(const std::vector& res) : m_resolution { res } { + Grid(const std::vector& res, const boundaries_t& ext) + : m_resolution { res } + , m_extent { ext } { raise::ErrorIf(m_resolution.size() != D, "invalid dimension", HERE); } + Grid(const std::vector& res, + const boundaries_t& ext, + const boundaries_t& flds_bc, + const boundaries_t& prtl_bc) + : Grid { res, ext } { + for (auto d { 0 }; d < D; ++d) { + dir::direction_t dir_plus; + dir_plus[d] = +1; + dir::direction_t dir_minus; + dir_minus[d] = -1; + set_flds_bc(dir_plus, flds_bc[d].second); + set_flds_bc(dir_minus, flds_bc[d].first); + set_prtl_bc(dir_plus, prtl_bc[d].second); + set_prtl_bc(dir_minus, prtl_bc[d].first); + } + } + ~Grid() = default; /* getters -------------------------------------------------------------- */ @@ -221,8 +243,66 @@ namespace ntt { */ auto rangeCellsOnHost(const box_region_t&) const -> range_h_t; + /* getters -------------------------------------------------------------- */ + [[nodiscard]] + auto extent(in i) const -> std::pair { + switch (i) { + case in::x1: + return (m_extent.size() > 0) ? m_extent[0] + : std::pair { ZERO, ZERO }; + case in::x2: + return (m_extent.size() > 1) ? m_extent[1] + : std::pair { ZERO, ZERO }; + case in::x3: + return (m_extent.size() > 2) ? m_extent[2] + : std::pair { ZERO, ZERO }; + default: + raise::Error("invalid dimension", HERE); + throw; + } + } + + [[nodiscard]] + auto extent() const -> boundaries_t { + return m_extent; + } + + [[nodiscard]] + auto flds_bc() const -> boundaries_t; + + [[nodiscard]] + auto prtl_bc() const -> boundaries_t; + + [[nodiscard]] + auto flds_bc_in(const dir::direction_t& direction) const -> FldsBC { + raise::ErrorIf(m_flds_bc.find(direction) == m_flds_bc.end(), + "direction not found", + HERE); + return m_flds_bc.at(direction); + } + + [[nodiscard]] + auto prtl_bc_in(const dir::direction_t& direction) const -> PrtlBC { + raise::ErrorIf(m_prtl_bc.find(direction) == m_prtl_bc.end(), + "direction not found", + HERE); + return m_prtl_bc.at(direction); + } + + /* setters -------------------------------------------------------------- */ + inline void set_flds_bc(const dir::direction_t& direction, const FldsBC& bc) { + m_flds_bc.insert_or_assign(direction, bc); + } + + inline void set_prtl_bc(const dir::direction_t& direction, const PrtlBC& bc) { + m_prtl_bc.insert_or_assign(direction, bc); + } + protected: std::vector m_resolution; + boundaries_t m_extent; + dir::map_t m_flds_bc; + dir::map_t m_prtl_bc; }; } // namespace ntt diff --git a/src/framework/domain/mesh.h b/src/framework/domain/mesh.h index 89f6a1b9f..cf7b29aa2 100644 --- a/src/framework/domain/mesh.h +++ b/src/framework/domain/mesh.h @@ -16,11 +16,12 @@ #include "enums.h" #include "global.h" -#include "arch/directions.h" #include "utils/comparators.h" #include "utils/error.h" #include "utils/numeric.h" +#include "metrics/traits.h" + #include "framework/domain/grid.h" #include @@ -31,39 +32,30 @@ namespace ntt { template + requires metric::traits::HasD && metric::traits::HasConvert_i struct Mesh : public Grid { - static_assert(M::is_metric, "template arg for Mesh class has to be a metric"); - static constexpr bool is_mesh { true }; static constexpr Dimension D { M::Dim }; + using base_t = Grid; + using base_t::extent; + using base_t::m_extent; + using base_t::m_flds_bc; + using base_t::m_prtl_bc; M metric; Mesh(const std::vector& res, const boundaries_t& ext, const std::map& metric_params) - : Grid { res } - , metric { res, ext, metric_params } - , m_extent { ext } {} + : Grid { res, ext } + , metric { res, ext, metric_params } {} Mesh(const std::vector& res, const boundaries_t& ext, const std::map& metric_params, const boundaries_t& flds_bc, const boundaries_t& prtl_bc) - : Grid { res } - , metric { res, ext, metric_params } - , m_extent { ext } { - for (auto d { 0 }; d < D; ++d) { - dir::direction_t dir_plus; - dir_plus[d] = +1; - dir::direction_t dir_minus; - dir_minus[d] = -1; - set_flds_bc(dir_plus, flds_bc[d].second); - set_flds_bc(dir_minus, flds_bc[d].first); - set_prtl_bc(dir_plus, prtl_bc[d].second); - set_prtl_bc(dir_minus, prtl_bc[d].first); - } - } + : Grid { res, ext, flds_bc, prtl_bc } + , metric { res, ext, metric_params } {} ~Mesh() = default; @@ -131,9 +123,8 @@ namespace ntt { * @note indices are already shifted by N_GHOSTS (i.e. they start at N_GHOSTS not 0) */ [[nodiscard]] - auto ExtentToRange( - boundaries_t box, - boundaries_t incl_ghosts) const -> boundaries_t { + auto ExtentToRange(boundaries_t box, boundaries_t incl_ghosts) const + -> boundaries_t { raise::ErrorIf(box.size() != M::Dim, "Invalid box dimension", HERE); raise::ErrorIf(incl_ghosts.size() != M::Dim, "Invalid incl_ghosts dimension", @@ -194,106 +185,6 @@ namespace ntt { } return range; } - - /* getters -------------------------------------------------------------- */ - [[nodiscard]] - auto extent(in i) const -> std::pair { - switch (i) { - case in::x1: - return (m_extent.size() > 0) ? m_extent[0] - : std::pair { ZERO, ZERO }; - case in::x2: - return (m_extent.size() > 1) ? m_extent[1] - : std::pair { ZERO, ZERO }; - case in::x3: - return (m_extent.size() > 2) ? m_extent[2] - : std::pair { ZERO, ZERO }; - default: - raise::Error("invalid dimension", HERE); - throw; - } - } - - [[nodiscard]] - auto extent() const -> boundaries_t { - return m_extent; - } - - [[nodiscard]] - auto flds_bc() const -> boundaries_t { - if constexpr (D == Dim::_1D) { - return { - { flds_bc_in({ -1 }), flds_bc_in({ 1 }) } - }; - } else if constexpr (D == Dim::_2D) { - return { - { flds_bc_in({ -1, 0 }), flds_bc_in({ 1, 0 }) }, - { flds_bc_in({ 0, -1 }), flds_bc_in({ 0, 1 }) } - }; - } else if constexpr (D == Dim::_3D) { - return { - { flds_bc_in({ -1, 0, 0 }), flds_bc_in({ 1, 0, 0 }) }, - { flds_bc_in({ 0, -1, 0 }), flds_bc_in({ 0, 1, 0 }) }, - { flds_bc_in({ 0, 0, -1 }), flds_bc_in({ 0, 0, 1 }) } - }; - } else { - raise::Error("invalid dimension", HERE); - throw; - } - } - - [[nodiscard]] - auto prtl_bc() const -> boundaries_t { - if constexpr (D == Dim::_1D) { - return { - { prtl_bc_in({ -1 }), prtl_bc_in({ 1 }) } - }; - } else if constexpr (D == Dim::_2D) { - return { - { prtl_bc_in({ -1, 0 }), prtl_bc_in({ 1, 0 }) }, - { prtl_bc_in({ 0, -1 }), prtl_bc_in({ 0, 1 }) } - }; - } else if constexpr (D == Dim::_3D) { - return { - { prtl_bc_in({ -1, 0, 0 }), prtl_bc_in({ 1, 0, 0 }) }, - { prtl_bc_in({ 0, -1, 0 }), prtl_bc_in({ 0, 1, 0 }) }, - { prtl_bc_in({ 0, 0, -1 }), prtl_bc_in({ 0, 0, 1 }) } - }; - } else { - raise::Error("invalid dimension", HERE); - throw; - } - } - - [[nodiscard]] - auto flds_bc_in(const dir::direction_t& direction) const -> FldsBC { - raise::ErrorIf(m_flds_bc.find(direction) == m_flds_bc.end(), - "direction not found", - HERE); - return m_flds_bc.at(direction); - } - - [[nodiscard]] - auto prtl_bc_in(const dir::direction_t& direction) const -> PrtlBC { - raise::ErrorIf(m_prtl_bc.find(direction) == m_prtl_bc.end(), - "direction not found", - HERE); - return m_prtl_bc.at(direction); - } - - /* setters -------------------------------------------------------------- */ - inline void set_flds_bc(const dir::direction_t& direction, const FldsBC& bc) { - m_flds_bc.insert_or_assign(direction, bc); - } - - inline void set_prtl_bc(const dir::direction_t& direction, const PrtlBC& bc) { - m_prtl_bc.insert_or_assign(direction, bc); - } - - private: - boundaries_t m_extent; - dir::map_t m_flds_bc; - dir::map_t m_prtl_bc; }; } // namespace ntt diff --git a/src/framework/domain/metadomain.cpp b/src/framework/domain/metadomain.cpp index cbea40843..e31015c24 100644 --- a/src/framework/domain/metadomain.cpp +++ b/src/framework/domain/metadomain.cpp @@ -26,6 +26,7 @@ namespace ntt { template + requires IsCompatibleWithMetadomain Metadomain::Metadomain(unsigned int global_ndomains, const std::vector& global_decomposition, const std::vector& global_ncells, @@ -57,6 +58,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::initialValidityCheck() const { // ensure everything has the correct shape raise::ErrorIf(g_decomposition.size() != (std::size_t)D, @@ -94,6 +96,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::createEmptyDomains() { /* decompose and compute cell & domain offsets ------------------------ */ auto d_ncells = tools::Decompose(g_ndomains, g_mesh.n_active(), g_decomposition); @@ -189,6 +192,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::redefineNeighbors() { for (unsigned int idx { 0 }; idx < g_ndomains; ++idx) { // offset of the subdomain[idx] @@ -229,6 +233,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::redefineBoundaries() { for (unsigned int idx { 0 }; idx < g_ndomains; ++idx) { // offset of the subdomain[idx] @@ -318,6 +323,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::finalValidityCheck() const { for (unsigned int idx { 0 }; idx < g_ndomains; ++idx) { const auto& current_domain = g_subdomains[idx]; @@ -363,6 +369,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::metricCompatibilityCheck() const { const auto epsilon = std::numeric_limits::epsilon() * static_cast(100.0); @@ -397,6 +404,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::setFldsBC(const bc_in& dir, const FldsBC& new_bcs) { if (dir == bc_in::Mx1) { if constexpr (M::Dim == Dim::_1D) { @@ -465,6 +473,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::setPrtlBC(const bc_in& dir, const PrtlBC& new_bcs) { if (dir == bc_in::Mx1) { if constexpr (M::Dim == Dim::_1D) { diff --git a/src/framework/domain/metadomain.h b/src/framework/domain/metadomain.h index a456879b4..d04433fcf 100644 --- a/src/framework/domain/metadomain.h +++ b/src/framework/domain/metadomain.h @@ -5,6 +5,10 @@ * - ntt::Metadomain<> * @cpp: * - metadomain.cpp + * - metadomain_comm.cpp + * - metadomain_chckpt.cpp + * - metadomain_io.cpp + * - metadomain_stats.cpp * @namespaces: * - ntt:: * @macros: @@ -20,10 +24,12 @@ #include "arch/kokkos_aliases.h" +#include "metrics/traits.h" + #include "framework/containers/species.h" #include "framework/domain/domain.h" #include "framework/domain/mesh.h" -#include "framework/parameters.h" +#include "framework/parameters/parameters.h" #include "output/stats.h" #if defined(MPI_ENABLED) @@ -35,7 +41,7 @@ #include "output/writer.h" #include - #include + #include #endif // OUTPUT_ENABLED #include @@ -46,10 +52,14 @@ namespace ntt { + template + concept IsCompatibleWithMetadomain = metric::traits::HasD && + metric::traits::HasConvert && + metric::traits::HasTotVolume; + template + requires IsCompatibleWithMetadomain struct Metadomain { - static_assert(M::is_metric, - "template arg for Metadomain class has to be a metric"); static constexpr Dimension D { M::Dim }; void initialValidityCheck() const; @@ -86,10 +96,15 @@ namespace ntt { } } - void CommunicateFields(Domain&, CommTags); - void SynchronizeFields(Domain&, CommTags, const range_tuple_t& = { 0, 0 }); - void CommunicateParticles(Domain&); - void RemoveDeadParticles(Domain&); + void CommunicateFields(Domain&, CommTags) const; + void SynchronizeFields(Domain&, + CommTags, + const range_tuple_t& = { 0, 0 }) const; + void CommunicateParticles(Domain&) const; + void SortParticles(simtime_t, + timestep_t, + const SimulationParams&, + Domain&) const; /** * @param global_ndomains total number of domains @@ -175,6 +190,12 @@ namespace ntt { return &g_subdomains[idx]; } + [[nodiscard]] + auto subdomain_ptr(unsigned int idx) const -> const Domain* { + raise::ErrorIf(idx >= g_subdomains.size(), "subdomain_ptr() failed", HERE); + return &g_subdomains[idx]; + } + [[nodiscard]] auto mesh() const -> const Mesh& { return g_mesh; diff --git a/src/framework/domain/checkpoint.cpp b/src/framework/domain/metadomain_chckpt.cpp similarity index 97% rename from src/framework/domain/checkpoint.cpp rename to src/framework/domain/metadomain_chckpt.cpp index cc4eac43e..bd6b87cfd 100644 --- a/src/framework/domain/checkpoint.cpp +++ b/src/framework/domain/metadomain_chckpt.cpp @@ -1,5 +1,3 @@ -#include "output/checkpoint.h" - #include "enums.h" #include "global.h" @@ -8,12 +6,14 @@ #include "utils/log.h" #include "framework/domain/metadomain.h" -#include "framework/parameters.h" +#include "framework/parameters/parameters.h" #include "framework/specialization_registry.h" +#include "output/checkpoint.h" namespace ntt { template + requires IsCompatibleWithMetadomain void Metadomain::InitCheckpointWriter(adios2::ADIOS* ptr_adios, const SimulationParams& params) { raise::ErrorIf(ptr_adios == nullptr, "adios == nullptr", HERE); @@ -64,6 +64,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain auto Metadomain::WriteCheckpoint(const SimulationParams& params, timestep_t current_step, timestep_t finished_step, @@ -110,6 +111,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::ContinueFromCheckpoint(adios2::ADIOS* ptr_adios, const SimulationParams& params) { raise::ErrorIf(ptr_adios == nullptr, "adios == nullptr", HERE); diff --git a/src/framework/domain/communications.cpp b/src/framework/domain/metadomain_comm.cpp similarity index 95% rename from src/framework/domain/communications.cpp rename to src/framework/domain/metadomain_comm.cpp index ae4ec83d1..45bf61013 100644 --- a/src/framework/domain/communications.cpp +++ b/src/framework/domain/metadomain_comm.cpp @@ -28,12 +28,12 @@ namespace ntt { using comm_params_t = std::pair>; template - auto GetSendRecvRanks( - Metadomain* metadomain, - Domain& domain, - dir::direction_t direction) -> std::pair { - Domain* send_to_nghbr_ptr = nullptr; - Domain* recv_from_nghbr_ptr = nullptr; + auto GetSendRecvRanks(const Metadomain* const metadomain, + Domain& domain, + dir::direction_t direction) + -> std::pair { + const Domain* send_to_nghbr_ptr = nullptr; + const Domain* recv_from_nghbr_ptr = nullptr; // set pointers to the correct send/recv domains // can coincide with the current domain if periodic if (domain.mesh.flds_bc_in(direction) == FldsBC::PERIODIC) { @@ -111,11 +111,11 @@ namespace ntt { } template - auto GetSendRecvParams( - Metadomain* metadomain, - Domain& domain, - dir::direction_t direction, - bool synchronize) -> std::pair { + auto GetSendRecvParams(const Metadomain* const metadomain, + Domain& domain, + dir::direction_t direction, + bool synchronize) + -> std::pair { const auto [send_indrank, recv_indrank] = GetSendRecvRanks(metadomain, domain, direction); const auto [send_ind, send_rank] = send_indrank; @@ -197,11 +197,9 @@ namespace ntt { } template - void Metadomain::CommunicateFields(Domain& domain, CommTags tags) { - // const auto comm_fields = (tags & Comm::E) or (tags & Comm::B) or - // (tags & Comm::J) or (tags & Comm::D) or - // (tags & Comm::D0) or (tags & Comm::B0) or - // (tags & Comm::H); + requires IsCompatibleWithMetadomain + void Metadomain::CommunicateFields(Domain& domain, + CommTags tags) const { const auto comm_em = ((S == SimEngine::SRPIC) and ((tags & Comm::E) or (tags & Comm::B))) or ((S == SimEngine::GRPIC) and @@ -417,9 +415,10 @@ namespace ntt { } template - void Metadomain::SynchronizeFields(Domain& domain, - CommTags tags, - const range_tuple_t& components) { + requires IsCompatibleWithMetadomain + void Metadomain::SynchronizeFields(Domain& domain, + CommTags tags, + const range_tuple_t& components) const { const bool comm_j = (tags & Comm::J); const bool comm_bckp = (tags & Comm::Bckp); const bool comm_buff = (tags & Comm::Buff); @@ -573,7 +572,8 @@ namespace ntt { } template - void Metadomain::CommunicateParticles(Domain& domain) { + requires IsCompatibleWithMetadomain + void Metadomain::CommunicateParticles(Domain& domain) const { #if defined(MPI_ENABLED) logger::Checkpoint("Communicating particles\n", HERE); for (auto& species : domain.species) { @@ -663,21 +663,14 @@ namespace ntt { #endif } - template - void Metadomain::RemoveDeadParticles(Domain& domain) { - for (auto& species : domain.species) { - species.RemoveDead(); - } - } - #define METADOMAIN_COMM(S, M, D) \ template void Metadomain>::CommunicateFields(Domain>&, \ - CommTags); \ + CommTags) const; \ template void Metadomain>::SynchronizeFields(Domain>&, \ CommTags, \ - const range_tuple_t&); \ - template void Metadomain>::CommunicateParticles(Domain>&); \ - template void Metadomain>::RemoveDeadParticles(Domain>&); + const range_tuple_t&) \ + const; \ + template void Metadomain>::CommunicateParticles(Domain>&) const; NTT_FOREACH_SPECIALIZATION(METADOMAIN_COMM) diff --git a/src/framework/domain/output.cpp b/src/framework/domain/metadomain_io.cpp similarity index 99% rename from src/framework/domain/output.cpp rename to src/framework/domain/metadomain_io.cpp index 4d625c4d0..b9f351a0f 100644 --- a/src/framework/domain/output.cpp +++ b/src/framework/domain/metadomain_io.cpp @@ -9,9 +9,8 @@ #include "framework/containers/particles.h" #include "framework/domain/domain.h" #include "framework/domain/metadomain.h" -#include "framework/parameters.h" +#include "framework/parameters/parameters.h" #include "framework/specialization_registry.h" - #include "kernels/divergences.hpp" #include "kernels/fields_to_phys.hpp" #include "kernels/particle_moments.hpp" @@ -31,6 +30,7 @@ namespace ntt { template + requires IsCompatibleWithMetadomain void Metadomain::InitWriter(adios2::ADIOS* ptr_adios, const SimulationParams& params) { raise::ErrorIf( @@ -57,8 +57,7 @@ namespace ntt { g_writer.init(ptr_adios, params.template get("output.format"), - params.template get("simulation.name"), - params.template get("output.separate_files")); + params.template get("simulation.name")); g_writer.defineMeshLayout(glob_shape_with_ghosts, off_ncells_with_ghosts, loc_shape_with_ghosts, @@ -81,10 +80,6 @@ namespace ntt { "output.particles.species"); g_writer.defineFieldOutputs(S, all_fields_to_write); - Dimension dim = M::PrtlDim; - if constexpr (M::CoordType != Coord::Cart) { - dim = Dim::_3D; - } g_writer.clearSpeciesIndex(); for (const auto& s : species_to_write) { g_writer.addSpeciesIndex(s); @@ -263,6 +258,7 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain void Metadomain::CommunicateVectorPotential(unsigned short buff_idx) { if constexpr (M::Dim == Dim::_2D) { auto local_domain = subdomain_ptr(l_subdomain_indices()[0]); @@ -280,7 +276,7 @@ namespace ntt { for (auto nr1 { 0u }; nr1 < nranks_x1; ++nr1) { const auto rank_send = rank_send_pre + nr1; const auto rank_recv = rank_recv_pre + nr1; - if (local_domain->mpi_rank() == rank_send) { + if (static_cast(local_domain->mpi_rank()) == rank_send) { array_t aphi_r { "Aphi_r", nx1 }; Kokkos::deep_copy( aphi_r, @@ -294,7 +290,8 @@ namespace ntt { rank_recv, 0, MPI_COMM_WORLD); - } else if (local_domain->mpi_rank() == rank_recv) { + } else if (static_cast(local_domain->mpi_rank()) == + rank_recv) { array_t aphi_r { "Aphi_r", nx1 }; MPI_Recv(aphi_r.data(), nx1, @@ -316,6 +313,7 @@ namespace ntt { #endif template + requires IsCompatibleWithMetadomain auto Metadomain::Write( const SimulationParams& params, timestep_t current_step, diff --git a/src/framework/domain/metadomain_sort.cpp b/src/framework/domain/metadomain_sort.cpp new file mode 100644 index 000000000..c529ca738 --- /dev/null +++ b/src/framework/domain/metadomain_sort.cpp @@ -0,0 +1,43 @@ +#include "enums.h" + +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" +#include "framework/specialization_registry.h" + +#include + +namespace ntt { + + template + requires IsCompatibleWithMetadomain + void Metadomain::SortParticles(simtime_t, + timestep_t step, + const SimulationParams&, + Domain& domain) const { + for (auto& species : domain.species) { + const auto clearing_interval = species.clearing_interval(); + if ((clearing_interval > 0u) and (step % clearing_interval == 0u) and + (step > 0u)) { + for (auto& species : domain.species) { + species.RemoveDead(); + } + } + const auto spatial_sorting_interval = species.spatial_sorting_interval(); + if ((spatial_sorting_interval > 0u) and + (step % spatial_sorting_interval == 0u)) { + species.SortSpatially(domain.mesh); + } + } + } + +#define METADOMAIN_COMM(S, M, D) \ + template void Metadomain>::SortParticles(simtime_t, \ + timestep_t, \ + const SimulationParams&, \ + Domain>&) const; + + NTT_FOREACH_SPECIALIZATION(METADOMAIN_COMM) + +#undef METADOMAIN_COMM + +} // namespace ntt diff --git a/src/framework/domain/stats.cpp b/src/framework/domain/metadomain_stats.cpp similarity index 96% rename from src/framework/domain/stats.cpp rename to src/framework/domain/metadomain_stats.cpp index 64690d695..21176a836 100644 --- a/src/framework/domain/stats.cpp +++ b/src/framework/domain/metadomain_stats.cpp @@ -9,9 +9,8 @@ #include "framework/containers/particles.h" #include "framework/domain/domain.h" #include "framework/domain/metadomain.h" -#include "framework/parameters.h" +#include "framework/parameters/parameters.h" #include "framework/specialization_registry.h" - #include "kernels/reduced_stats.hpp" #include @@ -23,6 +22,7 @@ namespace ntt { template + requires IsCompatibleWithMetadomain void Metadomain::InitStatsWriter(const SimulationParams& params, bool is_resuming) { raise::ErrorIf( @@ -182,14 +182,15 @@ namespace ntt { } template + requires IsCompatibleWithMetadomain auto Metadomain::WriteStats( const SimulationParams& params, timestep_t current_step, timestep_t finished_step, simtime_t current_time, simtime_t finished_time, - std::function&)> - CustomStat) -> bool { + std::function&)> CustomStat) + -> bool { if (not(params.template get("output.stats.enable") and g_stats_writer.shouldWrite(finished_step, finished_time))) { return false; @@ -279,17 +280,18 @@ namespace ntt { return true; } -#define METADOMAIN_STATS(S, M, D) \ - template void Metadomain>::InitStatsWriter(const SimulationParams&, \ - bool); \ - template auto Metadomain>::WriteStats( \ - const SimulationParams&, \ - timestep_t, \ - timestep_t, \ - simtime_t, \ - simtime_t, \ - std::function< \ - real_t(const std::string&, timestep_t, simtime_t, const Domain>&)>) -> bool; +#define METADOMAIN_STATS(S, M, D) \ + template void Metadomain>::InitStatsWriter(const SimulationParams&, \ + bool); \ + template auto Metadomain>::WriteStats( \ + const SimulationParams&, \ + timestep_t, \ + timestep_t, \ + simtime_t, \ + simtime_t, \ + std::function< \ + real_t(const std::string&, timestep_t, simtime_t, const Domain>&)>) \ + -> bool; NTT_FOREACH_SPECIALIZATION(METADOMAIN_STATS) diff --git a/src/framework/parameters.cpp b/src/framework/parameters.cpp deleted file mode 100644 index 550bb20e0..000000000 --- a/src/framework/parameters.cpp +++ /dev/null @@ -1,1120 +0,0 @@ -#include "framework/parameters.h" - -#include "defaults.h" -#include "enums.h" -#include "global.h" - -#include "utils/error.h" -#include "utils/formatting.h" -#include "utils/log.h" -#include "utils/numeric.h" -#include "utils/toml.h" - -#include "metrics/kerr_schild.h" -#include "metrics/kerr_schild_0.h" -#include "metrics/minkowski.h" -#include "metrics/qkerr_schild.h" -#include "metrics/qspherical.h" -#include "metrics/spherical.h" - -#include "framework/containers/species.h" - -#if defined(MPI_ENABLED) - #include -#endif - -#include -#include -#include -#include -#include -#include - -namespace ntt { - - template - auto get_dx0_V0( - const std::vector& resolution, - const boundaries_t& extent, - const std::map& params) -> std::pair { - const auto metric = M(resolution, extent, params); - const auto dx0 = metric.dxMin(); - coord_t x_corner { ZERO }; - for (auto d { 0u }; d < M::Dim; ++d) { - x_corner[d] = HALF; - } - const auto V0 = metric.sqrt_det_h(x_corner); - return { dx0, V0 }; - } - - /* - * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - * Parameters that must not be changed during the checkpoint restart - * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - */ - void SimulationParams::setImmutableParams(const toml::value& toml_data) { - /* [simulation] --------------------------------------------------------- */ - const auto engine_enum = SimEngine::pick( - fmt::toLower(toml::find(toml_data, "simulation", "engine")).c_str()); - set("simulation.engine", engine_enum); - - int default_ndomains = 1; -#if defined(MPI_ENABLED) - raise::ErrorIf(MPI_Comm_size(MPI_COMM_WORLD, &default_ndomains) != MPI_SUCCESS, - "MPI_Comm_size failed", - HERE); -#endif - const auto ndoms = toml::find_or(toml_data, - "simulation", - "domain", - "number", - default_ndomains); - set("simulation.domain.number", (unsigned int)ndoms); - - auto decomposition = toml::find_or>( - toml_data, - "simulation", - "domain", - "decomposition", - std::vector { -1, -1, -1 }); - promiseToDefine("simulation.domain.decomposition"); - - /* [grid] --------------------------------------------------------------- */ - const auto res = toml::find>(toml_data, - "grid", - "resolution"); - raise::ErrorIf(res.size() < 1 || res.size() > 3, - "invalid `grid.resolution`", - HERE); - set("grid.resolution", res); - const auto dim = static_cast(res.size()); - set("grid.dim", dim); - - if (decomposition.size() > dim) { - decomposition.erase(decomposition.begin() + (std::size_t)(dim), - decomposition.end()); - } - raise::ErrorIf(decomposition.size() != dim, - "invalid `simulation.domain.decomposition`", - HERE); - set("simulation.domain.decomposition", decomposition); - - auto extent = toml::find>>(toml_data, - "grid", - "extent"); - raise::ErrorIf(extent.size() < 1 || extent.size() > 3, - "invalid `grid.extent`", - HERE); - promiseToDefine("grid.extent"); - - /* [grid.metric] -------------------------------------------------------- */ - const auto metric_enum = Metric::pick( - fmt::toLower(toml::find(toml_data, "grid", "metric", "metric")) - .c_str()); - promiseToDefine("grid.metric.metric"); - std::string coord; - if (metric_enum == Metric::Minkowski) { - raise::ErrorIf(engine_enum != SimEngine::SRPIC, - "minkowski metric is only supported for SRPIC", - HERE); - coord = "cart"; - } else if (metric_enum == Metric::QKerr_Schild or - metric_enum == Metric::QSpherical) { - // quasi-spherical geometry - raise::ErrorIf(dim == Dim::_1D, - "not enough dimensions for qspherical geometry", - HERE); - raise::ErrorIf(dim == Dim::_3D, - "3D not implemented for qspherical geometry", - HERE); - coord = "qsph"; - set("grid.metric.qsph_r0", - toml::find_or(toml_data, "grid", "metric", "qsph_r0", defaults::qsph::r0)); - set("grid.metric.qsph_h", - toml::find_or(toml_data, "grid", "metric", "qsph_h", defaults::qsph::h)); - } else { - // spherical geometry - raise::ErrorIf(dim == Dim::_1D, - "not enough dimensions for spherical geometry", - HERE); - raise::ErrorIf(dim == Dim::_3D, - "3D not implemented for spherical geometry", - HERE); - coord = "sph"; - } - if ((engine_enum == SimEngine::GRPIC) && - (metric_enum != Metric::Kerr_Schild_0)) { - const auto ks_a = toml::find_or(toml_data, - "grid", - "metric", - "ks_a", - defaults::ks::a); - set("grid.metric.ks_a", ks_a); - set("grid.metric.ks_rh", ONE + math::sqrt(ONE - SQR(ks_a))); - } - const auto coord_enum = Coord::pick(coord.c_str()); - set("grid.metric.coord", coord_enum); - - /* [scales] ------------------------------------------------------------- */ - const auto larmor0 = toml::find(toml_data, "scales", "larmor0"); - const auto skindepth0 = toml::find(toml_data, "scales", "skindepth0"); - raise::ErrorIf(larmor0 <= ZERO || skindepth0 <= ZERO, - "larmor0 and skindepth0 must be positive", - HERE); - set("scales.larmor0", larmor0); - set("scales.skindepth0", skindepth0); - promiseToDefine("scales.dx0"); - promiseToDefine("scales.V0"); - promiseToDefine("scales.n0"); - promiseToDefine("scales.q0"); - set("scales.sigma0", SQR(skindepth0 / larmor0)); - set("scales.B0", ONE / larmor0); - set("scales.omegaB0", ONE / larmor0); - - /* [particles] ---------------------------------------------------------- */ - const auto ppc0 = toml::find(toml_data, "particles", "ppc0"); - set("particles.ppc0", ppc0); - raise::ErrorIf(ppc0 <= 0.0, "ppc0 must be positive", HERE); - set("particles.use_weights", - toml::find_or(toml_data, "particles", "use_weights", false)); - - /* [particles.species] -------------------------------------------------- */ - std::vector species; - const auto species_tab = toml::find_or(toml_data, - "particles", - "species", - toml::array {}); - set("particles.nspec", species_tab.size()); - - spidx_t idx = 1; - for (const auto& sp : species_tab) { - const auto label = toml::find_or(sp, - "label", - "s" + std::to_string(idx)); - const auto mass = toml::find(sp, "mass"); - const auto charge = toml::find(sp, "charge"); - raise::ErrorIf((charge != 0.0f) && (mass == 0.0f), - "mass of the charged species must be non-zero", - HERE); - const auto is_massless = (mass == 0.0f) && (charge == 0.0f); - const auto def_pusher = (is_massless ? defaults::ph_pusher - : defaults::em_pusher); - const auto maxnpart_real = toml::find(sp, "maxnpart"); - const auto maxnpart = static_cast(maxnpart_real); - auto pusher = toml::find_or(sp, "pusher", std::string(def_pusher)); - const auto npayloads_real = toml::find_or(sp, - "n_payloads_real", - static_cast(0)); - const auto use_tracking = toml::find_or(sp, "tracking", false); - auto npayloads_int = toml::find_or(sp, - "n_payloads_int", - static_cast(0)); - if (use_tracking) { -#if !defined(MPI_ENABLED) - npayloads_int += 1; -#else - npayloads_int += 2; -#endif - } - const auto cooling = toml::find_or(sp, "cooling", std::string("None")); - raise::ErrorIf((fmt::toLower(cooling) != "none") && is_massless, - "cooling is only applicable to massive particles", - HERE); - raise::ErrorIf((fmt::toLower(pusher) == "photon") && !is_massless, - "photon pusher is only applicable to massless particles", - HERE); - bool use_gca = false; - if (pusher.find(',') != std::string::npos) { - raise::ErrorIf(fmt::toLower(pusher.substr(pusher.find(',') + 1, - pusher.size())) != "gca", - "invalid pusher syntax", - HERE); - use_gca = true; - pusher = pusher.substr(0, pusher.find(',')); - } - const auto pusher_enum = PrtlPusher::pick(pusher.c_str()); - const auto cooling_enum = Cooling::pick(cooling.c_str()); - if (use_gca) { - raise::ErrorIf(engine_enum != SimEngine::SRPIC, - "GCA pushers are only supported for SRPIC", - HERE); - promiseToDefine("algorithms.gca.e_ovr_b_max"); - promiseToDefine("algorithms.gca.larmor_max"); - } - if (cooling_enum == Cooling::SYNCHROTRON) { - raise::ErrorIf(engine_enum != SimEngine::SRPIC, - "Synchrotron cooling is only supported for SRPIC", - HERE); - promiseToDefine("algorithms.synchrotron.gamma_rad"); - } - - if (cooling_enum == Cooling::COMPTON) { - raise::ErrorIf(engine_enum != SimEngine::SRPIC, - "Inverse Compton cooling is only supported for SRPIC", - HERE); - promiseToDefine("algorithms.compton.gamma_rad"); - } - - species.emplace_back(ParticleSpecies(idx, - label, - mass, - charge, - maxnpart, - pusher_enum, - use_tracking, - use_gca, - cooling_enum, - npayloads_real, - npayloads_int)); - idx += 1; - } - set("particles.species", species); - - /* inferred variables --------------------------------------------------- */ - // extent - if (extent.size() > dim) { - extent.erase(extent.begin() + (std::size_t)(dim), extent.end()); - } - raise::ErrorIf(extent[0].size() != 2, "invalid `grid.extent[0]`", HERE); - if (coord_enum != Coord::Cart) { - raise::ErrorIf(extent.size() > 1, - "invalid `grid.extent` for non-cartesian geometry", - HERE); - extent.push_back({ ZERO, constant::PI }); - if (dim == Dim::_3D) { - extent.push_back({ ZERO, TWO * constant::PI }); - } - } - raise::ErrorIf(extent.size() != dim, "invalid inferred `grid.extent`", HERE); - boundaries_t extent_pairwise; - for (auto d { 0u }; d < (dim_t)dim; ++d) { - raise::ErrorIf(extent[d].size() != 2, - fmt::format("invalid inferred `grid.extent[%d]`", d), - HERE); - extent_pairwise.push_back({ extent[d][0], extent[d][1] }); - } - set("grid.extent", extent_pairwise); - - // metric, dx0, V0, n0, q0 - { - boundaries_t ext; - for (const auto& e : extent) { - ext.push_back({ e[0], e[1] }); - } - std::map params; - if (coord_enum == Coord::Qsph) { - params["r0"] = get("grid.metric.qsph_r0"); - params["h"] = get("grid.metric.qsph_h"); - } - if ((engine_enum == SimEngine::GRPIC) && - (metric_enum != Metric::Kerr_Schild_0)) { - params["a"] = get("grid.metric.ks_a"); - } - set("grid.metric.params", params); - - std::pair dx0_V0; - if (metric_enum == Metric::Minkowski) { - if (dim == Dim::_1D) { - dx0_V0 = get_dx0_V0>(res, ext, params); - } else if (dim == Dim::_2D) { - dx0_V0 = get_dx0_V0>(res, ext, params); - } else { - dx0_V0 = get_dx0_V0>(res, ext, params); - } - } else if (metric_enum == Metric::Spherical) { - dx0_V0 = get_dx0_V0>(res, ext, params); - } else if (metric_enum == Metric::QSpherical) { - dx0_V0 = get_dx0_V0>(res, ext, params); - } else if (metric_enum == Metric::Kerr_Schild) { - dx0_V0 = get_dx0_V0>(res, ext, params); - } else if (metric_enum == Metric::Kerr_Schild_0) { - dx0_V0 = get_dx0_V0>(res, ext, params); - } else if (metric_enum == Metric::QKerr_Schild) { - dx0_V0 = get_dx0_V0>(res, ext, params); - } - auto [dx0, V0] = dx0_V0; - set("scales.dx0", dx0); - set("scales.V0", V0); - set("scales.n0", ppc0 / V0); - set("scales.q0", V0 / (ppc0 * SQR(skindepth0))); - - set("grid.metric.metric", metric_enum); - } - } - - /* - * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - * Parameters that may be changed during the checkpoint restart - * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - */ - void SimulationParams::setMutableParams(const toml::value& toml_data) { - const auto engine_enum = get("simulation.engine"); - const auto coord_enum = get("grid.metric.coord"); - const auto dim = get("grid.dim"); - const auto extent_pairwise = get>("grid.extent"); - - /* [simulation] --------------------------------------------------------- */ - set("simulation.name", - toml::find(toml_data, "simulation", "name")); - set("simulation.runtime", - toml::find(toml_data, "simulation", "runtime")); - - /* [grid.boundaraies] --------------------------------------------------- */ - auto flds_bc = toml::find>>( - toml_data, - "grid", - "boundaries", - "fields"); - { - raise::ErrorIf(flds_bc.size() < 1 || flds_bc.size() > 3, - "invalid `grid.boundaries.fields`", - HERE); - promiseToDefine("grid.boundaries.fields"); - auto atm_defined = false; - for (const auto& bcs : flds_bc) { - for (const auto& bc : bcs) { - if (fmt::toLower(bc) == "match") { - promiseToDefine("grid.boundaries.match.ds"); - } - if (fmt::toLower(bc) == "atmosphere") { - raise::ErrorIf(atm_defined, - "ATMOSPHERE is only allowed in one direction", - HERE); - atm_defined = true; - promiseToDefine("grid.boundaries.atmosphere.temperature"); - promiseToDefine("grid.boundaries.atmosphere.density"); - promiseToDefine("grid.boundaries.atmosphere.height"); - promiseToDefine("grid.boundaries.atmosphere.ds"); - promiseToDefine("grid.boundaries.atmosphere.species"); - promiseToDefine("grid.boundaries.atmosphere.g"); - } - } - } - } - - auto prtl_bc = toml::find>>( - toml_data, - "grid", - "boundaries", - "particles"); - { - raise::ErrorIf(prtl_bc.size() < 1 || prtl_bc.size() > 3, - "invalid `grid.boundaries.particles`", - HERE); - promiseToDefine("grid.boundaries.particles"); - auto atm_defined = false; - for (const auto& bcs : prtl_bc) { - for (const auto& bc : bcs) { - if (fmt::toLower(bc) == "absorb") { - promiseToDefine("grid.boundaries.absorb.ds"); - } - if (fmt::toLower(bc) == "atmosphere") { - raise::ErrorIf(atm_defined, - "ATMOSPHERE is only allowed in one direction", - HERE); - atm_defined = true; - promiseToDefine("grid.boundaries.atmosphere.temperature"); - promiseToDefine("grid.boundaries.atmosphere.density"); - promiseToDefine("grid.boundaries.atmosphere.height"); - promiseToDefine("grid.boundaries.atmosphere.ds"); - promiseToDefine("grid.boundaries.atmosphere.species"); - promiseToDefine("grid.boundaries.atmosphere.g"); - } - } - } - } - - /* [algorithms] --------------------------------------------------------- */ - set("algorithms.current_filters", - toml::find_or(toml_data, - "algorithms", - "current_filters", - defaults::current_filters)); - - /* [algorithms.deposit] ------------------------------------------------- */ - set("algorithms.deposit.enable", - toml::find_or(toml_data, "algorithms", "deposit", "enable", true)); - set("algorithms.deposit.order", - toml::find_or(toml_data, "algorithms", "deposit", "order", 1)); - - /* [algorithms.fieldsolver] --------------------------------------------- */ - set("algorithms.fieldsolver.enable", - toml::find_or(toml_data, "algorithms", "fieldsolver", "enable", true)); - - set("algorithms.fieldsolver.delta_x", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "delta_x", - defaults::fieldsolver::delta_x)); - set("algorithms.fieldsolver.delta_y", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "delta_y", - defaults::fieldsolver::delta_y)); - set("algorithms.fieldsolver.delta_z", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "delta_z", - defaults::fieldsolver::delta_z)); - set("algorithms.fieldsolver.beta_xy", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "beta_xy", - defaults::fieldsolver::beta_xy)); - set("algorithms.fieldsolver.beta_yx", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "beta_yx", - defaults::fieldsolver::beta_yx)); - set("algorithms.fieldsolver.beta_xz", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "beta_xz", - defaults::fieldsolver::beta_xz)); - set("algorithms.fieldsolver.beta_zx", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "beta_zx", - defaults::fieldsolver::beta_zx)); - set("algorithms.fieldsolver.beta_yz", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "beta_yz", - defaults::fieldsolver::beta_yz)); - set("algorithms.fieldsolver.beta_zy", - toml::find_or(toml_data, - "algorithms", - "fieldsolver", - "beta_zy", - defaults::fieldsolver::beta_zy)); - /* [algorithms.timestep] ------------------------------------------------ */ - set("algorithms.timestep.CFL", - toml::find_or(toml_data, "algorithms", "timestep", "CFL", defaults::cfl)); - set("algorithms.timestep.dt", - get("algorithms.timestep.CFL") * get("scales.dx0")); - set("algorithms.timestep.correction", - toml::find_or(toml_data, - "algorithms", - "timestep", - "correction", - defaults::correction)); - - /* [algorithms.gr] ------------------------------------------------------ */ - if (engine_enum == SimEngine::GRPIC) { - set("algorithms.gr.pusher_eps", - toml::find_or(toml_data, - "algorithms", - "gr", - "pusher_eps", - defaults::gr::pusher_eps)); - set("algorithms.gr.pusher_niter", - toml::find_or(toml_data, - "algorithms", - "gr", - "pusher_niter", - defaults::gr::pusher_niter)); - } - /* [particles] ---------------------------------------------------------- */ - set("particles.clear_interval", - toml::find_or(toml_data, "particles", "clear_interval", defaults::clear_interval)); - const auto species_tab = toml::find_or(toml_data, - "particles", - "species", - toml::array {}); - std::vector species = get>( - "particles.species"); - raise::ErrorIf(species_tab.size() != species.size(), - "number of species changed after restart", - HERE); - - std::vector new_species; - - spidx_t idxM1 = 0; - for (const auto& sp : species_tab) { - const auto maxnpart_real = toml::find(sp, "maxnpart"); - const auto maxnpart = static_cast(maxnpart_real); - const auto particle_species = species[idxM1]; - new_species.emplace_back(particle_species.index(), - particle_species.label(), - particle_species.mass(), - particle_species.charge(), - maxnpart, - particle_species.pusher(), - particle_species.use_tracking(), - particle_species.use_gca(), - particle_species.cooling(), - particle_species.npld_r(), - particle_species.npld_i()); - idxM1++; - } - set("particles.species", new_species); - - /* [output] ------------------------------------------------------------- */ - // fields - set("output.format", - toml::find_or(toml_data, "output", "format", defaults::output::format)); - set("output.interval", - toml::find_or(toml_data, "output", "interval", defaults::output::interval)); - set("output.interval_time", - toml::find_or(toml_data, "output", "interval_time", -1.0)); - set("output.separate_files", - toml::find_or(toml_data, "output", "separate_files", true)); - - promiseToDefine("output.fields.enable"); - promiseToDefine("output.fields.interval"); - promiseToDefine("output.fields.interval_time"); - promiseToDefine("output.particles.enable"); - promiseToDefine("output.particles.interval"); - promiseToDefine("output.particles.interval_time"); - promiseToDefine("output.spectra.enable"); - promiseToDefine("output.spectra.interval"); - promiseToDefine("output.spectra.interval_time"); - promiseToDefine("output.stats.enable"); - promiseToDefine("output.stats.interval"); - promiseToDefine("output.stats.interval_time"); - - const auto flds_out = toml::find_or(toml_data, - "output", - "fields", - "quantities", - std::vector {}); - const auto custom_flds_out = toml::find_or(toml_data, - "output", - "fields", - "custom", - std::vector {}); - if (flds_out.size() == 0) { - raise::Warning("No fields output specified", HERE); - } - set("output.fields.quantities", flds_out); - set("output.fields.custom", custom_flds_out); - set("output.fields.mom_smooth", - toml::find_or(toml_data, - "output", - "fields", - "mom_smooth", - defaults::output::mom_smooth)); - std::vector field_dwn; - try { - auto field_dwn_ = toml::find>(toml_data, - "output", - "fields", - "downsampling"); - for (auto i = 0u; i < field_dwn_.size(); ++i) { - field_dwn.push_back(field_dwn_[i]); - } - } catch (...) { - try { - auto field_dwn_ = toml::find(toml_data, - "output", - "fields", - "downsampling"); - for (auto i = 0u; i < dim; ++i) { - field_dwn.push_back(field_dwn_); - } - } catch (...) { - for (auto i = 0u; i < dim; ++i) { - field_dwn.push_back(1u); - } - } - } - raise::ErrorIf(field_dwn.size() > 3, "invalid `output.fields.downsampling`", HERE); - if (field_dwn.size() > dim) { - field_dwn.erase(field_dwn.begin() + (std::size_t)(dim), field_dwn.end()); - } - for (const auto& dwn : field_dwn) { - raise::ErrorIf(dwn == 0, "downsampling factor must be nonzero", HERE); - } - set("output.fields.downsampling", field_dwn); - - // particles - auto all_specs = std::vector {}; - const auto nspec = get("particles.nspec"); - for (auto i = 0u; i < nspec; ++i) { - all_specs.push_back(static_cast(i + 1)); - } - const auto prtl_out = toml::find_or(toml_data, - "output", - "particles", - "species", - all_specs); - set("output.particles.species", prtl_out); - set("output.particles.stride", - toml::find_or(toml_data, - "output", - "particles", - "stride", - defaults::output::prtl_stride)); - - // spectra - set("output.spectra.e_min", - toml::find_or(toml_data, "output", "spectra", "e_min", defaults::output::spec_emin)); - set("output.spectra.e_max", - toml::find_or(toml_data, "output", "spectra", "e_max", defaults::output::spec_emax)); - set("output.spectra.log_bins", - toml::find_or(toml_data, - "output", - "spectra", - "log_bins", - defaults::output::spec_log)); - set("output.spectra.n_bins", - toml::find_or(toml_data, - "output", - "spectra", - "n_bins", - defaults::output::spec_nbins)); - - // stats - set("output.stats.quantities", - toml::find_or(toml_data, - "output", - "stats", - "quantities", - defaults::output::stats_quantities)); - set("output.stats.custom", - toml::find_or(toml_data, - "output", - "stats", - "custom", - std::vector {})); - - // intervals - for (const auto& type : { "fields", "particles", "spectra", "stats" }) { - const auto q_int = toml::find_or(toml_data, - "output", - std::string(type), - "interval", - 0); - const auto q_int_time = toml::find_or(toml_data, - "output", - std::string(type), - "interval_time", - -1.0); - set("output." + std::string(type) + ".enable", - toml::find_or(toml_data, "output", std::string(type), "enable", true)); - if ((q_int == 0) and (q_int_time == -1.0)) { - set("output." + std::string(type) + ".interval", - get("output.interval")); - set("output." + std::string(type) + ".interval_time", - get("output.interval_time")); - } else { - set("output." + std::string(type) + ".interval", q_int); - set("output." + std::string(type) + ".interval_time", q_int_time); - } - } - - /* [output.debug] ------------------------------------------------------- */ - set("output.debug.as_is", - toml::find_or(toml_data, "output", "debug", "as_is", false)); - const auto output_ghosts = toml::find_or(toml_data, - "output", - "debug", - "ghosts", - false); - set("output.debug.ghosts", output_ghosts); - if (output_ghosts) { - for (const auto& dwn : field_dwn) { - raise::ErrorIf( - dwn != 1, - "full resolution required when outputting with ghost cells", - HERE); - } - } - - /* [checkpoint] --------------------------------------------------------- */ - set("checkpoint.interval", - toml::find_or(toml_data, - "checkpoint", - "interval", - defaults::checkpoint::interval)); - set("checkpoint.interval_time", - toml::find_or(toml_data, "checkpoint", "interval_time", -1.0)); - set("checkpoint.keep", - toml::find_or(toml_data, "checkpoint", "keep", defaults::checkpoint::keep)); - auto walltime_str = toml::find_or(toml_data, - "checkpoint", - "walltime", - defaults::checkpoint::walltime); - if (walltime_str.empty()) { - walltime_str = defaults::checkpoint::walltime; - } - set("checkpoint.walltime", walltime_str); - - const auto checkpoint_write_path = toml::find_or( - toml_data, - "checkpoint", - "write_path", - fmt::format(defaults::checkpoint::write_path.c_str(), - get("simulation.name").c_str())); - set("checkpoint.write_path", checkpoint_write_path); - set("checkpoint.read_path", - toml::find_or(toml_data, "checkpoint", "read_path", checkpoint_write_path)); - - /* [diagnostics] -------------------------------------------------------- */ - set("diagnostics.interval", - toml::find_or(toml_data, "diagnostics", "interval", defaults::diag::interval)); - set("diagnostics.blocking_timers", - toml::find_or(toml_data, "diagnostics", "blocking_timers", false)); - set("diagnostics.colored_stdout", - toml::find_or(toml_data, "diagnostics", "colored_stdout", false)); - set("diagnostics.log_level", - toml::find_or(toml_data, "diagnostics", "log_level", defaults::diag::log_level)); - - /* inferred variables --------------------------------------------------- */ - // fields/particle boundaries - std::vector> flds_bc_enum; - std::vector> prtl_bc_enum; - if (coord_enum == Coord::Cart) { - raise::ErrorIf(flds_bc.size() != (std::size_t)dim, - "invalid `grid.boundaries.fields`", - HERE); - raise::ErrorIf(prtl_bc.size() != (std::size_t)dim, - "invalid `grid.boundaries.particles`", - HERE); - for (auto d { 0u }; d < (dim_t)dim; ++d) { - flds_bc_enum.push_back({}); - prtl_bc_enum.push_back({}); - const auto fbc = flds_bc[d]; - const auto pbc = prtl_bc[d]; - raise::ErrorIf(fbc.size() < 1 || fbc.size() > 2, - "invalid `grid.boundaries.fields`", - HERE); - raise::ErrorIf(pbc.size() < 1 || pbc.size() > 2, - "invalid `grid.boundaries.particles`", - HERE); - auto fbc_enum = FldsBC::pick(fmt::toLower(fbc[0]).c_str()); - auto pbc_enum = PrtlBC::pick(fmt::toLower(pbc[0]).c_str()); - if (fbc.size() == 1) { - raise::ErrorIf(fbc_enum != FldsBC::PERIODIC, - "invalid `grid.boundaries.fields`", - HERE); - flds_bc_enum.back().push_back(FldsBC(FldsBC::PERIODIC)); - flds_bc_enum.back().push_back(FldsBC(FldsBC::PERIODIC)); - } else { - raise::ErrorIf(fbc_enum == FldsBC::PERIODIC, - "invalid `grid.boundaries.fields`", - HERE); - flds_bc_enum.back().push_back(fbc_enum); - auto fbc_enum = FldsBC::pick(fmt::toLower(fbc[1]).c_str()); - raise::ErrorIf(fbc_enum == FldsBC::PERIODIC, - "invalid `grid.boundaries.fields`", - HERE); - flds_bc_enum.back().push_back(fbc_enum); - } - if (pbc.size() == 1) { - raise::ErrorIf(pbc_enum != PrtlBC::PERIODIC, - "invalid `grid.boundaries.particles`", - HERE); - prtl_bc_enum.back().push_back(PrtlBC(PrtlBC::PERIODIC)); - prtl_bc_enum.back().push_back(PrtlBC(PrtlBC::PERIODIC)); - } else { - raise::ErrorIf(pbc_enum == PrtlBC::PERIODIC, - "invalid `grid.boundaries.particles`", - HERE); - prtl_bc_enum.back().push_back(pbc_enum); - auto pbc_enum = PrtlBC::pick(fmt::toLower(pbc[1]).c_str()); - raise::ErrorIf(pbc_enum == PrtlBC::PERIODIC, - "invalid `grid.boundaries.particles`", - HERE); - prtl_bc_enum.back().push_back(pbc_enum); - } - } - } else { - raise::ErrorIf(flds_bc.size() > 1, "invalid `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc.size() > 1, "invalid `grid.boundaries.particles`", HERE); - if (engine_enum == SimEngine::SRPIC) { - raise::ErrorIf(flds_bc[0].size() != 2, - "invalid `grid.boundaries.fields`", - HERE); - flds_bc_enum.push_back( - { FldsBC::pick(fmt::toLower(flds_bc[0][0]).c_str()), - FldsBC::pick(fmt::toLower(flds_bc[0][1]).c_str()) }); - flds_bc_enum.push_back({ FldsBC::AXIS, FldsBC::AXIS }); - if (dim == Dim::_3D) { - flds_bc_enum.push_back({ FldsBC::PERIODIC, FldsBC::PERIODIC }); - } - raise::ErrorIf(prtl_bc[0].size() != 2, - "invalid `grid.boundaries.particles`", - HERE); - prtl_bc_enum.push_back( - { PrtlBC::pick(fmt::toLower(prtl_bc[0][0]).c_str()), - PrtlBC::pick(fmt::toLower(prtl_bc[0][1]).c_str()) }); - prtl_bc_enum.push_back({ PrtlBC::AXIS, PrtlBC::AXIS }); - if (dim == Dim::_3D) { - prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); - } - } else { - raise::ErrorIf(flds_bc[0].size() != 1, - "invalid `grid.boundaries.fields`", - HERE); - raise::ErrorIf(prtl_bc[0].size() != 1, - "invalid `grid.boundaries.particles`", - HERE); - flds_bc_enum.push_back( - { FldsBC::HORIZON, FldsBC::pick(fmt::toLower(flds_bc[0][0]).c_str()) }); - flds_bc_enum.push_back({ FldsBC::AXIS, FldsBC::AXIS }); - if (dim == Dim::_3D) { - flds_bc_enum.push_back({ FldsBC::PERIODIC, FldsBC::PERIODIC }); - } - prtl_bc_enum.push_back( - { PrtlBC::HORIZON, PrtlBC::pick(fmt::toLower(prtl_bc[0][0]).c_str()) }); - prtl_bc_enum.push_back({ PrtlBC::AXIS, PrtlBC::AXIS }); - if (dim == Dim::_3D) { - prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); - } - } - } - - raise::ErrorIf(flds_bc_enum.size() != (std::size_t)dim, - "invalid inferred `grid.boundaries.fields`", - HERE); - raise::ErrorIf(prtl_bc_enum.size() != (std::size_t)dim, - "invalid inferred `grid.boundaries.particles`", - HERE); - boundaries_t flds_bc_pairwise; - boundaries_t prtl_bc_pairwise; - for (auto d { 0u }; d < (dim_t)dim; ++d) { - raise::ErrorIf( - flds_bc_enum[d].size() != 2, - fmt::format("invalid inferred `grid.boundaries.fields[%d]`", d), - HERE); - raise::ErrorIf( - prtl_bc_enum[d].size() != 2, - fmt::format("invalid inferred `grid.boundaries.particles[%d]`", d), - HERE); - flds_bc_pairwise.push_back({ flds_bc_enum[d][0], flds_bc_enum[d][1] }); - prtl_bc_pairwise.push_back({ prtl_bc_enum[d][0], prtl_bc_enum[d][1] }); - } - set("grid.boundaries.fields", flds_bc_pairwise); - set("grid.boundaries.particles", prtl_bc_pairwise); - - if (isPromised("grid.boundaries.match.ds")) { - if (coord_enum == Coord::Cart) { - auto min_extent = std::numeric_limits::max(); - for (const auto& e : extent_pairwise) { - min_extent = std::min(min_extent, e.second - e.first); - } - const auto default_ds = min_extent * defaults::bc::match::ds_frac; - boundaries_t ds_array; - try { - auto ds = toml::find(toml_data, "grid", "boundaries", "match", "ds"); - for (auto d = 0u; d < dim; ++d) { - ds_array.push_back({ ds, ds }); - } - } catch (...) { - try { - const auto ds = toml::find>>( - toml_data, - "grid", - "boundaries", - "match", - "ds"); - raise::ErrorIf(ds.size() != dim, - "invalid # in `grid.boundaries.match.ds`", - HERE); - for (auto d = 0u; d < dim; ++d) { - if (ds[d].size() == 1) { - ds_array.push_back({ ds[d][0], ds[d][0] }); - } else if (ds[d].size() == 2) { - ds_array.push_back({ ds[d][0], ds[d][1] }); - } else if (ds[d].size() == 0) { - ds_array.push_back({}); - } else { - raise::Error("invalid `grid.boundaries.match.ds`", HERE); - } - } - } catch (...) { - for (auto d = 0u; d < dim; ++d) { - ds_array.push_back({ default_ds, default_ds }); - } - } - } - set("grid.boundaries.match.ds", ds_array); - } else { - auto r_extent = extent_pairwise[0].second - extent_pairwise[0].first; - const auto ds = toml::find_or( - toml_data, - "grid", - "boundaries", - "match", - "ds", - r_extent * defaults::bc::match::ds_frac); - boundaries_t ds_array { - { ds, ds } - }; - set("grid.boundaries.match.ds", ds_array); - } - } - - if (isPromised("grid.boundaries.absorb.ds")) { - if (coord_enum == Coord::Cart) { - auto min_extent = std::numeric_limits::max(); - for (const auto& e : extent_pairwise) { - min_extent = std::min(min_extent, e.second - e.first); - } - set("grid.boundaries.absorb.ds", - toml::find_or(toml_data, - "grid", - "boundaries", - "absorb", - "ds", - min_extent * defaults::bc::absorb::ds_frac)); - } else { - auto r_extent = extent_pairwise[0].second - extent_pairwise[0].first; - set("grid.boundaries.absorb.ds", - toml::find_or(toml_data, - "grid", - "boundaries", - "absorb", - "ds", - r_extent * defaults::bc::absorb::ds_frac)); - } - } - - if (isPromised("grid.boundaries.atmosphere.temperature")) { - const auto atm_T = toml::find(toml_data, - "grid", - "boundaries", - "atmosphere", - "temperature"); - const auto atm_h = toml::find(toml_data, - "grid", - "boundaries", - "atmosphere", - "height"); - set("grid.boundaries.atmosphere.temperature", atm_T); - set("grid.boundaries.atmosphere.density", - toml::find(toml_data, "grid", "boundaries", "atmosphere", "density")); - set("grid.boundaries.atmosphere.ds", - toml::find_or(toml_data, "grid", "boundaries", "atmosphere", "ds", ZERO)); - set("grid.boundaries.atmosphere.height", atm_h); - set("grid.boundaries.atmosphere.g", atm_T / atm_h); - const auto atm_species = toml::find>( - toml_data, - "grid", - "boundaries", - "atmosphere", - "species"); - set("grid.boundaries.atmosphere.species", atm_species); - } - - // gca - if (isPromised("algorithms.gca.e_ovr_b_max")) { - set("algorithms.gca.e_ovr_b_max", - toml::find_or(toml_data, - "algorithms", - "gca", - "e_ovr_b_max", - defaults::gca::EovrB_max)); - set("algorithms.gca.larmor_max", - toml::find_or(toml_data, "algorithms", "gca", "larmor_max", ZERO)); - } - - // cooling - if (isPromised("algorithms.synchrotron.gamma_rad")) { - set("algorithms.synchrotron.gamma_rad", - toml::find_or(toml_data, - "algorithms", - "synchrotron", - "gamma_rad", - defaults::synchrotron::gamma_rad)); - } - if (isPromised("algorithms.compton.gamma_rad")) { - set("algorithms.compton.gamma_rad", - toml::find_or(toml_data, - "algorithms", - "compton", - "gamma_rad", - defaults::compton::gamma_rad)); - } - - // @TODO: disabling stats for non-Cartesian - if (coord_enum != Coord::Cart) { - set("output.stats.enable", false); - } - } - - void SimulationParams::setSetupParams(const toml::value& toml_data) { - /* [setup] -------------------------------------------------------------- */ - const auto setup = toml::find_or(toml_data, "setup", toml::table {}); - for (const auto& [key, val] : setup) { - if (val.is_boolean()) { - set("setup." + key, (bool)(val.as_boolean())); - } else if (val.is_integer()) { - set("setup." + key, (int)(val.as_integer())); - } else if (val.is_floating()) { - set("setup." + key, (real_t)(val.as_floating())); - } else if (val.is_string()) { - set("setup." + key, (std::string)(val.as_string())); - } else if (val.is_array()) { - const auto val_arr = val.as_array(); - if (val_arr.size() == 0) { - continue; - } else { - if (val_arr[0].is_integer()) { - std::vector vec; - for (const auto& v : val_arr) { - vec.push_back(v.as_integer()); - } - set("setup." + key, vec); - } else if (val_arr[0].is_floating()) { - std::vector vec; - for (const auto& v : val_arr) { - vec.push_back(v.as_floating()); - } - set("setup." + key, vec); - } else if (val_arr[0].is_boolean()) { - std::vector vec; - for (const auto& v : val_arr) { - vec.push_back(v.as_boolean()); - } - set("setup." + key, vec); - } else if (val_arr[0].is_string()) { - std::vector vec; - for (const auto& v : val_arr) { - vec.push_back(v.as_string()); - } - set("setup." + key, vec); - } else if (val_arr[0].is_array()) { - raise::Error("only 1D arrays allowed in [setup]", HERE); - } else { - raise::Error("invalid setup variable type", HERE); - } - } - } - } - } - - void SimulationParams::setCheckpointParams(bool is_resuming, - timestep_t start_step, - simtime_t start_time) { - set("checkpoint.is_resuming", is_resuming); - set("checkpoint.start_step", start_step); - set("checkpoint.start_time", start_time); - } - - void SimulationParams::checkPromises() const { - raise::ErrorIf(!promisesFulfilled(), - "Have not defined all the necessary variables", - HERE); - } - - void SimulationParams::saveTOML(const std::string& path, simtime_t time) const { - CallOnce([&]() { - std::ofstream metadata; - metadata.open(path); - metadata << fmt::format("[metadata]\n time = %f\n\n", time) << data() - << std::endl; - metadata.close(); - }); - } - -} // namespace ntt diff --git a/src/framework/parameters/algorithms.cpp b/src/framework/parameters/algorithms.cpp new file mode 100644 index 000000000..856d05fc7 --- /dev/null +++ b/src/framework/parameters/algorithms.cpp @@ -0,0 +1,158 @@ +#include "framework/parameters/algorithms.h" + +#include "defaults.h" +#include "global.h" + +#include "utils/numeric.h" + +#include "framework/parameters/parameters.h" + +#include + +namespace ntt { + namespace params { + + void Algorithms::read(real_t dx0, + const std::map& extra, + const toml::value& toml_data) { + CFL = toml::find_or(toml_data, "algorithms", "timestep", "CFL", defaults::cfl); + dt = CFL * dx0; + dt_correction_factor = toml::find_or(toml_data, + "algorithms", + "timestep", + "correction", + defaults::correction); + + number_of_current_filters = toml::find_or(toml_data, + "algorithms", + "current_filters", + defaults::current_filters); + + deposit_enable = toml::find_or(toml_data, "algorithms", "deposit", "enable", true); + deposit_order = static_cast(SHAPE_ORDER); + + fieldsolver_enable = toml::find_or(toml_data, + "algorithms", + "fieldsolver", + "enable", + true); + + fieldsolver_stencil_coeffs["delta_x"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "delta_x", + defaults::fieldsolver::delta_x); + + fieldsolver_stencil_coeffs["delta_y"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "delta_y", + defaults::fieldsolver::delta_y); + + fieldsolver_stencil_coeffs["delta_z"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "delta_z", + defaults::fieldsolver::delta_z); + + fieldsolver_stencil_coeffs["beta_xy"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "beta_xy", + defaults::fieldsolver::beta_xy); + + fieldsolver_stencil_coeffs["beta_yx"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "beta_yx", + defaults::fieldsolver::beta_yx); + + fieldsolver_stencil_coeffs["beta_xz"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "beta_xz", + defaults::fieldsolver::beta_xz); + + fieldsolver_stencil_coeffs["beta_zx"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "beta_zx", + defaults::fieldsolver::beta_zx); + + fieldsolver_stencil_coeffs["beta_yz"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "beta_yz", + defaults::fieldsolver::beta_yz); + + fieldsolver_stencil_coeffs["beta_zy"] = toml::find_or( + toml_data, + "algorithms", + "fieldsolver", + "beta_zy", + defaults::fieldsolver::beta_zy); + + if (extra.at("gr")) { + gr_pusher_eps = toml::find_or(toml_data, + "algorithms", + "gr", + "pusher_eps", + defaults::gr::pusher_eps); + gr_pusher_niter = toml::find_or(toml_data, + "algorithms", + "gr", + "pusher_niter", + defaults::gr::pusher_niter); + } + + if (extra.at("gca")) { + gca_e_ovr_b_max = toml::find_or(toml_data, + "algorithms", + "gca", + "e_ovr_b_max", + defaults::gca::EovrB_max); + gca_larmor_max = toml::find_or(toml_data, + "algorithms", + "gca", + "larmor_max", + ZERO); + } + } + + void Algorithms::setParams(const std::map& extra, + SimulationParams* params) const { + params->set("algorithms.timestep.CFL", CFL); + params->set("algorithms.timestep.dt", dt); + params->set("algorithms.timestep.correction", dt_correction_factor); + + params->set("algorithms.current_filters", number_of_current_filters); + + params->set("algorithms.deposit.enable", deposit_enable); + params->set("algorithms.deposit.order", deposit_order); + + params->set("algorithms.fieldsolver.enable", fieldsolver_enable); + for (const auto& [key, value] : fieldsolver_stencil_coeffs) { + params->set("algorithms.fieldsolver." + key, value); + } + + if (extra.at("gr")) { + params->set("algorithms.gr.pusher_eps", gr_pusher_eps); + params->set("algorithms.gr.pusher_niter", gr_pusher_niter); + } + + if (extra.at("gca")) { + params->set("algorithms.gca.e_ovr_b_max", gca_e_ovr_b_max); + params->set("algorithms.gca.larmor_max", gca_larmor_max); + } + } + + } // namespace params +} // namespace ntt diff --git a/src/framework/parameters/algorithms.h b/src/framework/parameters/algorithms.h new file mode 100644 index 000000000..a496cb7b6 --- /dev/null +++ b/src/framework/parameters/algorithms.h @@ -0,0 +1,56 @@ +/** + * @file framework/parameters/algorithms.h + * @brief Auxiliary functions for reading in algorithms parameters + * @implements + * - ntt::params::Algorithms + * @cpp: + * - algorithms.cpp + * @namespaces: + * - ntt::params:: + */ + +#ifndef FRAMEWORK_PARAMETERS_ALGORITHMS_H +#define FRAMEWORK_PARAMETERS_ALGORITHMS_H + +#include "global.h" + +#include "framework/parameters/parameters.h" + +#include + +#include +#include + +namespace ntt { + namespace params { + + struct Algorithms { + real_t CFL; + real_t dt; + real_t dt_correction_factor; + + unsigned short number_of_current_filters; + + bool deposit_enable; + unsigned short deposit_order; + + bool fieldsolver_enable; + std::map fieldsolver_stencil_coeffs; + + real_t gr_pusher_eps; + unsigned short gr_pusher_niter; + + real_t gca_e_ovr_b_max; + real_t gca_larmor_max; + + real_t synchrotron_gamma_rad; + real_t compton_gamma_rad; + + void read(real_t, const std::map&, const toml::value&); + void setParams(const std::map&, SimulationParams*) const; + }; + + } // namespace params +} // namespace ntt + +#endif // FRAMEWORK_PARAMETERS_ALGORITHMS_H diff --git a/src/framework/parameters/extra.cpp b/src/framework/parameters/extra.cpp new file mode 100644 index 000000000..3e8c2dcd9 --- /dev/null +++ b/src/framework/parameters/extra.cpp @@ -0,0 +1,153 @@ +#include "framework/parameters/extra.h" + +#include "defaults.h" + +#include "utils/numeric.h" + +namespace ntt { + namespace params { + + void Extra::read(const std::map& extra, + const toml::value& toml_data, + const SimulationParams* const params) { + if (extra.at("synchrotron_drag")) { + synchrotron_gamma_rad = toml::find_or(toml_data, + "radiation", + "drag", + "synchrotron", + "gamma_rad", + defaults::synchrotron::gamma_rad); + } + + if (extra.at("compton_drag")) { + compton_gamma_rad = toml::find_or(toml_data, + "radiation", + "drag", + "compton", + "gamma_rad", + defaults::compton::gamma_rad); + } + + if (extra.at("synchrotron_emission")) { + synchrotron_gamma_rad = toml::find_or(toml_data, + "radiation", + "drag", + "synchrotron", + "gamma_rad", + defaults::synchrotron::gamma_rad); + synchrotron_gamma_qed = toml::find_or(toml_data, + "radiation", + "emission", + "synchrotron", + "gamma_qed", + defaults::synchrotron::gamma_qed); + synchrotron_energy_min = toml::find_or(toml_data, + "radiation", + "emission", + "synchrotron", + "photon_energy_min", + defaults::synchrotron::energy_min); + synchrotron_photon_weight = toml::find_or(toml_data, + "radiation", + "emission", + "synchrotron", + "photon_weight", + ONE); + synchrotron_photon_species = toml::find(toml_data, + "radiation", + "emission", + "synchrotron", + "photon_species"); + synchrotron_nominal_probability = + params->template get("scales.omegaB0") * + static_cast(0.1) * + params->template get("algorithms.timestep.dt") * + SQR(synchrotron_gamma_qed / synchrotron_gamma_rad) / + synchrotron_photon_weight; + synchrotron_nominal_photon_energy = ONE / SQR(synchrotron_gamma_qed); + } + + if (extra.at("compton_emission")) { + compton_gamma_rad = toml::find_or(toml_data, + "radiation", + "drag", + "compton", + "gamma_rad", + defaults::compton::gamma_rad); + compton_gamma_qed = toml::find_or(toml_data, + "radiation", + "emission", + "compton", + "gamma_qed", + defaults::compton::gamma_qed); + compton_energy_min = toml::find_or(toml_data, + "radiation", + "emission", + "compton", + "photon_energy_min", + defaults::compton::energy_min); + compton_photon_weight = toml::find_or(toml_data, + "radiation", + "emission", + "compton", + "photon_weight", + ONE); + compton_photon_species = toml::find(toml_data, + "radiation", + "emission", + "compton", + "photon_species"); + compton_nominal_probability = params->template get( + "scales.omegaB0") * + static_cast(0.1) * + params->template get( + "algorithms.timestep.dt") * + SQR(compton_gamma_qed / compton_gamma_rad) / + compton_photon_weight; + compton_nominal_photon_energy = ONE / SQR(compton_gamma_qed); + } + } + + void Extra::setParams(const std::map& extra, + SimulationParams* params) const { + if (extra.at("synchrotron_drag")) { + params->set("radiation.drag.synchrotron.gamma_rad", synchrotron_gamma_rad); + } + + if (extra.at("compton_drag")) { + params->set("radiation.drag.compton.gamma_rad", compton_gamma_rad); + } + + if (extra.at("synchrotron_emission")) { + params->set("radiation.drag.synchrotron.gamma_rad", synchrotron_gamma_rad); + params->set("radiation.emission.synchrotron.gamma_qed", + synchrotron_gamma_qed); + params->set("radiation.emission.synchrotron.photon_energy_min", + synchrotron_energy_min); + params->set("radiation.emission.synchrotron.photon_weight", + synchrotron_photon_weight); + params->set("radiation.emission.synchrotron.photon_species", + synchrotron_photon_species); + params->set("radiation.emission.synchrotron.nominal_probability", + synchrotron_nominal_probability); + params->set("radiation.emission.synchrotron.nominal_photon_energy", + synchrotron_nominal_photon_energy); + } + + if (extra.at("compton_emission")) { + params->set("radiation.drag.compton.gamma_rad", compton_gamma_rad); + params->set("radiation.emission.compton.gamma_qed", compton_gamma_qed); + params->set("radiation.emission.compton.photon_energy_min", + compton_energy_min); + params->set("radiation.emission.compton.photon_weight", + compton_photon_weight); + params->set("radiation.emission.compton.photon_species", + compton_photon_species); + params->set("radiation.emission.compton.nominal_probability", + compton_nominal_probability); + params->set("radiation.emission.compton.nominal_photon_energy", + compton_nominal_photon_energy); + } + } + } // namespace params +} // namespace ntt diff --git a/src/framework/parameters/extra.h b/src/framework/parameters/extra.h new file mode 100644 index 000000000..cf5c0849c --- /dev/null +++ b/src/framework/parameters/extra.h @@ -0,0 +1,56 @@ +/** + * @file framework/parameters/algorithms.h + * @brief Auxiliary functions for reading in extra physics parameters + * @implements + * - ntt::params::Extra + * @cpp: + * - extra.cpp + * @namespaces: + * - ntt::params:: + */ + +#ifndef FRAMEWORK_PARAMETERS_EXTRA_H +#define FRAMEWORK_PARAMETERS_EXTRA_H + +#include "global.h" + +#include "framework/parameters/parameters.h" + +#include + +#include +#include + +namespace ntt { + namespace params { + + struct Extra { + // radiative drag parameters + real_t synchrotron_gamma_rad; + real_t compton_gamma_rad; + + // emission parameters + real_t synchrotron_energy_min; + real_t synchrotron_gamma_qed; + real_t synchrotron_photon_weight; + spidx_t synchrotron_photon_species; + real_t synchrotron_nominal_probability; + real_t synchrotron_nominal_photon_energy; + + real_t compton_energy_min; + real_t compton_gamma_qed; + real_t compton_photon_weight; + spidx_t compton_photon_species; + real_t compton_nominal_probability; + real_t compton_nominal_photon_energy; + + void read(const std::map&, + const toml::value&, + const SimulationParams* const); + void setParams(const std::map&, SimulationParams*) const; + }; + + } // namespace params +} // namespace ntt + +#endif // FRAMEWORK_PARAMETERS_EXTRA_H diff --git a/src/framework/parameters/grid.cpp b/src/framework/parameters/grid.cpp new file mode 100644 index 000000000..9e5d39df5 --- /dev/null +++ b/src/framework/parameters/grid.cpp @@ -0,0 +1,552 @@ +#include "framework/parameters/grid.h" + +#include "defaults.h" +#include "global.h" + +#include "utils/error.h" +#include "utils/formatting.h" +#include "utils/numeric.h" + +#include "metrics/kerr_schild.h" +#include "metrics/kerr_schild_0.h" +#include "metrics/minkowski.h" +#include "metrics/qkerr_schild.h" +#include "metrics/qspherical.h" +#include "metrics/spherical.h" + +#include "framework/parameters/parameters.h" + +#include + +#include +#include +#include +#include + +namespace ntt { + namespace params { + + template + auto get_dx0_V0(const std::vector& resolution, + const boundaries_t& extent, + const std::map& params) + -> std::pair { + const auto metric = M(resolution, extent, params); + const auto dx0 = metric.dxMin(); + coord_t x_corner { ZERO }; + for (auto d { 0u }; d < M::Dim; ++d) { + x_corner[d] = HALF; + } + const auto V0 = metric.sqrt_det_h(x_corner); + return { dx0, V0 }; + } + + auto GetBoundaryConditions(SimulationParams* params, + const SimEngine& engine_enum, + Dimension dim, + const Coord& coord_enum, + const toml::value& toml_data) + -> std::tuple, boundaries_t> { + auto flds_bc = toml::find>>( + toml_data, + "grid", + "boundaries", + "fields"); + { + raise::ErrorIf(flds_bc.size() < 1 || flds_bc.size() > 3, + "invalid `grid.boundaries.fields`", + HERE); + params->promiseToDefine("grid.boundaries.fields"); + auto atm_defined = false; + for (const auto& bcs : flds_bc) { + for (const auto& bc : bcs) { + if (fmt::toLower(bc) == "match") { + params->promiseToDefine("grid.boundaries.match.ds"); + } + if (fmt::toLower(bc) == "atmosphere") { + raise::ErrorIf(atm_defined, + "ATMOSPHERE is only allowed in one direction", + HERE); + atm_defined = true; + params->promiseToDefine("grid.boundaries.atmosphere.temperature"); + params->promiseToDefine("grid.boundaries.atmosphere.density"); + params->promiseToDefine("grid.boundaries.atmosphere.height"); + params->promiseToDefine("grid.boundaries.atmosphere.ds"); + params->promiseToDefine("grid.boundaries.atmosphere.species"); + params->promiseToDefine("grid.boundaries.atmosphere.g"); + } + } + } + } + + auto prtl_bc = toml::find>>( + toml_data, + "grid", + "boundaries", + "particles"); + { + raise::ErrorIf(prtl_bc.size() < 1 || prtl_bc.size() > 3, + "invalid `grid.boundaries.particles`", + HERE); + params->promiseToDefine("grid.boundaries.particles"); + auto atm_defined = false; + for (const auto& bcs : prtl_bc) { + for (const auto& bc : bcs) { + if (fmt::toLower(bc) == "absorb") { + params->promiseToDefine("grid.boundaries.absorb.ds"); + } + if (fmt::toLower(bc) == "atmosphere") { + raise::ErrorIf(atm_defined, + "ATMOSPHERE is only allowed in one direction", + HERE); + atm_defined = true; + params->promiseToDefine("grid.boundaries.atmosphere.temperature"); + params->promiseToDefine("grid.boundaries.atmosphere.density"); + params->promiseToDefine("grid.boundaries.atmosphere.height"); + params->promiseToDefine("grid.boundaries.atmosphere.ds"); + params->promiseToDefine("grid.boundaries.atmosphere.species"); + params->promiseToDefine("grid.boundaries.atmosphere.g"); + } + } + } + } + std::vector> flds_bc_enum; + std::vector> prtl_bc_enum; + if (coord_enum == Coord::Cart) { + raise::ErrorIf(flds_bc.size() != (std::size_t)dim, + "invalid `grid.boundaries.fields`", + HERE); + raise::ErrorIf(prtl_bc.size() != (std::size_t)dim, + "invalid `grid.boundaries.particles`", + HERE); + for (auto d { 0u }; d < (dim_t)dim; ++d) { + flds_bc_enum.push_back({}); + prtl_bc_enum.push_back({}); + const auto fbc = flds_bc[d]; + const auto pbc = prtl_bc[d]; + raise::ErrorIf(fbc.size() < 1 || fbc.size() > 2, + "invalid `grid.boundaries.fields`", + HERE); + raise::ErrorIf(pbc.size() < 1 || pbc.size() > 2, + "invalid `grid.boundaries.particles`", + HERE); + auto fbc_enum = FldsBC::pick(fmt::toLower(fbc[0]).c_str()); + auto pbc_enum = PrtlBC::pick(fmt::toLower(pbc[0]).c_str()); + if (fbc.size() == 1) { + raise::ErrorIf(fbc_enum != FldsBC::PERIODIC, + "invalid `grid.boundaries.fields`", + HERE); + flds_bc_enum.back().push_back(FldsBC(FldsBC::PERIODIC)); + flds_bc_enum.back().push_back(FldsBC(FldsBC::PERIODIC)); + } else { + raise::ErrorIf(fbc_enum == FldsBC::PERIODIC, + "invalid `grid.boundaries.fields`", + HERE); + flds_bc_enum.back().push_back(fbc_enum); + auto fbc_enum = FldsBC::pick(fmt::toLower(fbc[1]).c_str()); + raise::ErrorIf(fbc_enum == FldsBC::PERIODIC, + "invalid `grid.boundaries.fields`", + HERE); + flds_bc_enum.back().push_back(fbc_enum); + } + if (pbc.size() == 1) { + raise::ErrorIf(pbc_enum != PrtlBC::PERIODIC, + "invalid `grid.boundaries.particles`", + HERE); + prtl_bc_enum.back().push_back(PrtlBC(PrtlBC::PERIODIC)); + prtl_bc_enum.back().push_back(PrtlBC(PrtlBC::PERIODIC)); + } else { + raise::ErrorIf(pbc_enum == PrtlBC::PERIODIC, + "invalid `grid.boundaries.particles`", + HERE); + prtl_bc_enum.back().push_back(pbc_enum); + auto pbc_enum = PrtlBC::pick(fmt::toLower(pbc[1]).c_str()); + raise::ErrorIf(pbc_enum == PrtlBC::PERIODIC, + "invalid `grid.boundaries.particles`", + HERE); + prtl_bc_enum.back().push_back(pbc_enum); + } + } + } else { + raise::ErrorIf(flds_bc.size() > 1, "invalid `grid.boundaries.fields`", HERE); + raise::ErrorIf(prtl_bc.size() > 1, + "invalid `grid.boundaries.particles`", + HERE); + if (engine_enum == SimEngine::SRPIC) { + raise::ErrorIf(flds_bc[0].size() != 2, + "invalid `grid.boundaries.fields`", + HERE); + flds_bc_enum.push_back( + { FldsBC::pick(fmt::toLower(flds_bc[0][0]).c_str()), + FldsBC::pick(fmt::toLower(flds_bc[0][1]).c_str()) }); + flds_bc_enum.push_back({ FldsBC::AXIS, FldsBC::AXIS }); + if (dim == Dim::_3D) { + flds_bc_enum.push_back({ FldsBC::PERIODIC, FldsBC::PERIODIC }); + } + raise::ErrorIf(prtl_bc[0].size() != 2, + "invalid `grid.boundaries.particles`", + HERE); + prtl_bc_enum.push_back( + { PrtlBC::pick(fmt::toLower(prtl_bc[0][0]).c_str()), + PrtlBC::pick(fmt::toLower(prtl_bc[0][1]).c_str()) }); + prtl_bc_enum.push_back({ PrtlBC::AXIS, PrtlBC::AXIS }); + if (dim == Dim::_3D) { + prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); + } + } else { + raise::ErrorIf(flds_bc[0].size() != 1, + "invalid `grid.boundaries.fields`", + HERE); + raise::ErrorIf(prtl_bc[0].size() != 1, + "invalid `grid.boundaries.particles`", + HERE); + flds_bc_enum.push_back( + { FldsBC::HORIZON, FldsBC::pick(fmt::toLower(flds_bc[0][0]).c_str()) }); + flds_bc_enum.push_back({ FldsBC::AXIS, FldsBC::AXIS }); + if (dim == Dim::_3D) { + flds_bc_enum.push_back({ FldsBC::PERIODIC, FldsBC::PERIODIC }); + } + prtl_bc_enum.push_back( + { PrtlBC::HORIZON, PrtlBC::pick(fmt::toLower(prtl_bc[0][0]).c_str()) }); + prtl_bc_enum.push_back({ PrtlBC::AXIS, PrtlBC::AXIS }); + if (dim == Dim::_3D) { + prtl_bc_enum.push_back({ PrtlBC::PERIODIC, PrtlBC::PERIODIC }); + } + } + } + + raise::ErrorIf(flds_bc_enum.size() != (std::size_t)dim, + "invalid inferred `grid.boundaries.fields`", + HERE); + raise::ErrorIf(prtl_bc_enum.size() != (std::size_t)dim, + "invalid inferred `grid.boundaries.particles`", + HERE); + boundaries_t flds_bc_pairwise; + boundaries_t prtl_bc_pairwise; + for (auto d { 0u }; d < (dim_t)dim; ++d) { + raise::ErrorIf( + flds_bc_enum[d].size() != 2, + fmt::format("invalid inferred `grid.boundaries.fields[%d]`", d), + HERE); + raise::ErrorIf( + prtl_bc_enum[d].size() != 2, + fmt::format("invalid inferred `grid.boundaries.particles[%d]`", d), + HERE); + flds_bc_pairwise.push_back({ flds_bc_enum[d][0], flds_bc_enum[d][1] }); + prtl_bc_pairwise.push_back({ prtl_bc_enum[d][0], prtl_bc_enum[d][1] }); + } + return { flds_bc_pairwise, prtl_bc_pairwise }; + } + + void Boundaries::read(Dimension dim, + const Coord& coord_enum, + const boundaries_t& extent_pairwise, + const toml::value& toml_data) { + if (needs_match_boundaries) { + if (coord_enum == Coord::Cart) { + auto min_extent = std::numeric_limits::max(); + for (const auto& e : extent_pairwise) { + min_extent = std::min(min_extent, e.second - e.first); + } + const auto default_ds = min_extent * defaults::bc::match::ds_frac; + try { + auto ds = toml::find(toml_data, "grid", "boundaries", "match", "ds"); + for (auto d = 0u; d < dim; ++d) { + match_ds_array.push_back({ ds, ds }); + } + } catch (...) { + try { + const auto ds = toml::find>>( + toml_data, + "grid", + "boundaries", + "match", + "ds"); + raise::ErrorIf(ds.size() != dim, + "invalid # in `grid.boundaries.match.ds`", + HERE); + for (auto d = 0u; d < dim; ++d) { + if (ds[d].size() == 1) { + match_ds_array.push_back({ ds[d][0], ds[d][0] }); + } else if (ds[d].size() == 2) { + match_ds_array.push_back({ ds[d][0], ds[d][1] }); + } else if (ds[d].size() == 0) { + match_ds_array.push_back({}); + } else { + raise::Error("invalid `grid.boundaries.match.ds`", HERE); + } + } + } catch (...) { + for (auto d = 0u; d < dim; ++d) { + match_ds_array.push_back({ default_ds, default_ds }); + } + } + } + } else { + auto r_extent = extent_pairwise[0].second - extent_pairwise[0].first; + const auto ds = toml::find_or( + toml_data, + "grid", + "boundaries", + "match", + "ds", + r_extent * defaults::bc::match::ds_frac); + match_ds_array.push_back({ ds, ds }); + } + } + + if (needs_absorb_boundaries) { + if (coord_enum == Coord::Cart) { + auto min_extent = std::numeric_limits::max(); + for (const auto& e : extent_pairwise) { + min_extent = std::min(min_extent, e.second - e.first); + } + absorb_ds = toml::find_or(toml_data, + "grid", + "boundaries", + "absorb", + "ds", + min_extent * defaults::bc::absorb::ds_frac); + } else { + auto r_extent = extent_pairwise[0].second - extent_pairwise[0].first; + absorb_ds = toml::find_or(toml_data, + "grid", + "boundaries", + "absorb", + "ds", + r_extent * defaults::bc::absorb::ds_frac); + } + } + + if (needs_atmosphere_boundaries) { + atmosphere_temperature = toml::find(toml_data, + "grid", + "boundaries", + "atmosphere", + "temperature"); + atmosphere_height = toml::find(toml_data, + "grid", + "boundaries", + "atmosphere", + "height"); + atmosphere_density = toml::find(toml_data, + "grid", + "boundaries", + "atmosphere", + "density"); + atmosphere_ds = + toml::find_or(toml_data, "grid", "boundaries", "atmosphere", "ds", ZERO); + atmosphere_g = atmosphere_temperature / atmosphere_height; + atmosphere_species = toml::find>( + toml_data, + "grid", + "boundaries", + "atmosphere", + "species"); + } + } + + void Boundaries::setParams(SimulationParams* params) const { + if (needs_match_boundaries) { + params->set("grid.boundaries.match.ds", match_ds_array); + } + if (needs_absorb_boundaries) { + params->set("grid.boundaries.absorb.ds", absorb_ds); + } + if (needs_atmosphere_boundaries) { + params->set("grid.boundaries.atmosphere.temperature", + atmosphere_temperature); + params->set("grid.boundaries.atmosphere.density", atmosphere_density); + params->set("grid.boundaries.atmosphere.height", atmosphere_height); + params->set("grid.boundaries.atmosphere.ds", atmosphere_ds); + params->set("grid.boundaries.atmosphere.g", atmosphere_g); + params->set("grid.boundaries.atmosphere.species", atmosphere_species); + } + } + + void Grid::read(const SimEngine& engine_enum, const toml::value& toml_data) { + /* domain decomposition ------------------------------------------------ */ + int default_ndomains = 1; +#if defined(MPI_ENABLED) + raise::ErrorIf(MPI_Comm_size(MPI_COMM_WORLD, &default_ndomains) != MPI_SUCCESS, + "MPI_Comm_size failed", + HERE); +#endif + number_of_domains = toml::find_or(toml_data, + "simulation", + "domain", + "number", + (unsigned int)default_ndomains); + + domain_decomposition = toml::find_or>( + toml_data, + "simulation", + "domain", + "decomposition", + std::vector { -1, -1, -1 }); + + /* resolution and dimension ------------------------------------------- */ + resolution = toml::find>(toml_data, "grid", "resolution"); + raise::ErrorIf(resolution.size() < 1 || resolution.size() > 3, + "invalid `grid.resolution`", + HERE); + dim = static_cast(resolution.size()); + + if (domain_decomposition.size() > dim) { + domain_decomposition.erase(domain_decomposition.begin() + (std::size_t)(dim), + domain_decomposition.end()); + } + raise::ErrorIf(domain_decomposition.size() != dim, + "invalid `simulation.domain.decomposition`", + HERE); + + /* metric and coordinates -------------------------------------------- */ + metric_enum = Metric::pick( + fmt::toLower(toml::find(toml_data, "grid", "metric", "metric")) + .c_str()); + std::string coord; + if (metric_enum == Metric::Minkowski) { + raise::ErrorIf(engine_enum != SimEngine::SRPIC, + "minkowski metric is only supported for SRPIC", + HERE); + coord = "cart"; + } else if (metric_enum == Metric::QKerr_Schild or + metric_enum == Metric::QSpherical) { + // quasi-spherical geometry + raise::ErrorIf(dim == Dim::_1D, + "not enough dimensions for qspherical geometry", + HERE); + raise::ErrorIf(dim == Dim::_3D, + "3D not implemented for qspherical geometry", + HERE); + coord = "qsph"; + metric_params["qsph_r0"] = toml::find_or(toml_data, + "grid", + "metric", + "qsph_r0", + defaults::qsph::r0); + metric_params["qsph_h"] = toml::find_or(toml_data, + "grid", + "metric", + "qsph_h", + defaults::qsph::h); + } else { + // spherical geometry + raise::ErrorIf(dim == Dim::_1D, + "not enough dimensions for spherical geometry", + HERE); + raise::ErrorIf(dim == Dim::_3D, + "3D not implemented for spherical geometry", + HERE); + coord = "sph"; + } + if ((engine_enum == SimEngine::GRPIC) && + (metric_enum != Metric::Kerr_Schild_0)) { + const auto ks_a = toml::find_or(toml_data, + "grid", + "metric", + "ks_a", + defaults::ks::a); + metric_params["ks_a"] = ks_a; + metric_params["ks_rh"] = ONE + math::sqrt(ONE - SQR(ks_a)); + } + coord_enum = Coord::pick(coord.c_str()); + + /* extent ------------------------------------------------------------- */ + extent = toml::find>>(toml_data, + "grid", + "extent"); + + if (extent.size() > dim) { + extent.erase(extent.begin() + (std::size_t)(dim), extent.end()); + } + raise::ErrorIf(extent[0].size() != 2, "invalid `grid.extent[0]`", HERE); + if (coord_enum != Coord::Cart) { + raise::ErrorIf(extent.size() > 1, + "invalid `grid.extent` for non-cartesian geometry", + HERE); + extent.push_back({ ZERO, constant::PI }); + if (dim == Dim::_3D) { + extent.push_back({ ZERO, TWO * constant::PI }); + } + } + raise::ErrorIf(extent.size() != dim, "invalid inferred `grid.extent`", HERE); + for (auto d { 0u }; d < (dim_t)dim; ++d) { + raise::ErrorIf(extent[d].size() != 2, + fmt::format("invalid inferred `grid.extent[%d]`", d), + HERE); + extent_pairwise_.push_back({ extent[d][0], extent[d][1] }); + } + + /* metric parameters ------------------------------------------------------ */ + if (coord_enum == Coord::Qsph) { + metric_params_short_["r0"] = metric_params["qsph_r0"]; + metric_params_short_["h"] = metric_params["qsph_h"]; + } + if ((engine_enum == SimEngine::GRPIC) && + (metric_enum != Metric::Kerr_Schild_0)) { + metric_params_short_["a"] = metric_params["ks_a"]; + } + // set("grid.metric.params", params); + + std::pair dx0_V0; + if (metric_enum == Metric::Minkowski) { + if (dim == Dim::_1D) { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } else if (dim == Dim::_2D) { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } else { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } + } else if (metric_enum == Metric::Spherical) { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } else if (metric_enum == Metric::QSpherical) { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } else if (metric_enum == Metric::Kerr_Schild) { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } else if (metric_enum == Metric::Kerr_Schild_0) { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } else if (metric_enum == Metric::QKerr_Schild) { + dx0_V0 = get_dx0_V0>(resolution, + extent_pairwise_, + metric_params_short_); + } + auto [dx0, V0] = dx0_V0; + scale_dx0 = dx0; + scale_V0 = V0; + } + + void Grid::setParams(SimulationParams* params) const { + params->set("simulation.domain.number", number_of_domains); + params->set("simulation.domain.decomposition", domain_decomposition); + + params->set("grid.resolution", resolution); + params->set("grid.dim", dim); + params->set("grid.metric.metric", metric_enum); + params->set("grid.metric.coord", coord_enum); + for (const auto& [key, value] : metric_params) { + params->set("grid.metric." + key, value); + } + params->set("grid.metric.params", metric_params_short_); + params->set("grid.extent", extent_pairwise_); + + params->set("scales.dx0", scale_dx0); + params->set("scales.V0", scale_V0); + } + + } // namespace params +} // namespace ntt diff --git a/src/framework/parameters/grid.h b/src/framework/parameters/grid.h new file mode 100644 index 000000000..978b7f329 --- /dev/null +++ b/src/framework/parameters/grid.h @@ -0,0 +1,88 @@ +/** + * @file framework/parameters/grid.h + * @brief Auxiliary functions for reading in grid/box parameters + * @implements + * - ntt::params::Boundaries + * - ntt::params::GetBoundaryConditions -> (boundaries_t, boundaries_t) + * @cpp: + * - grid.cpp + * @namespaces: + * - ntt::params:: + */ +#ifndef FRAMEWORK_PARAMETERS_GRID_H +#define FRAMEWORK_PARAMETERS_GRID_H + +#include "enums.h" +#include "global.h" + +#include "framework/parameters/parameters.h" + +#include + +#include +#include +#include + +namespace ntt { + namespace params { + + struct Boundaries { + const bool needs_match_boundaries; + boundaries_t match_ds_array; + + const bool needs_absorb_boundaries; + real_t absorb_ds; + + const bool needs_atmosphere_boundaries; + real_t atmosphere_temperature; + real_t atmosphere_height; + real_t atmosphere_density; + real_t atmosphere_g; + real_t atmosphere_ds; + std::pair atmosphere_species; + + Boundaries(bool needs_match, bool needs_absorb, bool needs_atmosphere) + : needs_match_boundaries { needs_match } + , needs_absorb_boundaries { needs_absorb } + , needs_atmosphere_boundaries { needs_atmosphere } {} + + void read(Dimension, + const Coord&, + const boundaries_t&, + const toml::value&); + void setParams(SimulationParams*) const; + }; + + struct Grid { + unsigned int number_of_domains; + std::vector domain_decomposition; + + std::vector resolution; + Dimension dim; + + std::vector> extent; + boundaries_t extent_pairwise_; + + Metric metric_enum = Metric::INVALID; + Coord coord_enum = Coord::INVALID; + std::map metric_params; + std::map metric_params_short_; + + real_t scale_dx0; + real_t scale_V0; + + void read(const SimEngine&, const toml::value&); + void setParams(SimulationParams*) const; + }; + + auto GetBoundaryConditions(SimulationParams* params, + const SimEngine&, + Dimension, + const Coord&, + const toml::value&) + -> std::tuple, boundaries_t>; + + } // namespace params +} // namespace ntt + +#endif // FRAMEWORK_PARAMETERS_GRID_H diff --git a/src/framework/parameters/output.cpp b/src/framework/parameters/output.cpp new file mode 100644 index 000000000..547800971 --- /dev/null +++ b/src/framework/parameters/output.cpp @@ -0,0 +1,204 @@ +#include "framework/parameters/output.h" + +#include "defaults.h" +#include "global.h" + +#include "utils/error.h" +#include "utils/log.h" + +#include + +namespace ntt { + namespace params { + + void Output::read(Dimension dim, std::size_t nspec, const toml::value& toml_data) { + format = toml::find_or(toml_data, "output", "format", defaults::output::format); + global_interval = toml::find_or(toml_data, + "output", + "interval", + defaults::output::interval); + global_interval_time = toml::find_or(toml_data, + "output", + "interval_time", + -1.0); + raise::ErrorIf( + not toml::find_or(toml_data, "output", "separate_files", true), + "separate_files=false is deprecated", + HERE); + + for (const auto& category : { "fields", "particles", "spectra", "stats" }) { + const auto q_int = toml::find_or(toml_data, + "output", + category, + "interval", + 0); + const auto q_int_time = toml::find_or(toml_data, + "output", + category, + "interval_time", + -1.0); + categories[category].enable = toml::find_or(toml_data, + "output", + category, + "enable", + true); + if ((q_int == 0) and (q_int_time == -1.0)) { + categories[category].interval = global_interval; + categories[category].interval_time = global_interval_time; + } else { + categories[category].interval = q_int; + categories[category].interval_time = q_int_time; + } + } + + /* Fields --------------------------------------------------------------- */ + const auto flds_out = toml::find_or(toml_data, + "output", + "fields", + "quantities", + std::vector {}); + const auto custom_flds_out = toml::find_or(toml_data, + "output", + "fields", + "custom", + std::vector {}); + if (flds_out.size() == 0) { + raise::Warning("No fields output specified", HERE); + } + fields_quantities = flds_out; + fields_custom_quantities = custom_flds_out; + fields_mom_smooth = toml::find_or(toml_data, + "output", + "fields", + "mom_smooth", + defaults::output::mom_smooth); + try { + auto field_dwn_ = toml::find>(toml_data, + "output", + "fields", + "downsampling"); + for (auto i = 0u; i < field_dwn_.size(); ++i) { + fields_downsampling.push_back(field_dwn_[i]); + } + } catch (...) { + try { + auto field_dwn_ = toml::find(toml_data, + "output", + "fields", + "downsampling"); + for (auto i = 0u; i < dim; ++i) { + fields_downsampling.push_back(field_dwn_); + } + } catch (...) { + for (auto i = 0u; i < dim; ++i) { + fields_downsampling.push_back(1u); + } + } + } + raise::ErrorIf(fields_downsampling.size() > 3, + "invalid `output.fields.downsampling`", + HERE); + if (fields_downsampling.size() > dim) { + fields_downsampling.erase(fields_downsampling.begin() + (std::size_t)(dim), + fields_downsampling.end()); + } + for (const auto& dwn : fields_downsampling) { + raise::ErrorIf(dwn == 0, "downsampling factor must be nonzero", HERE); + } + + /* Particles ------------------------------------------------------------ */ + auto all_specs = std::vector {}; + for (auto i = 0u; i < nspec; ++i) { + all_specs.push_back(static_cast(i + 1)); + } + particles_species = toml::find_or(toml_data, + "output", + "particles", + "species", + all_specs); + particles_stride = toml::find_or(toml_data, + "output", + "particles", + "stride", + defaults::output::prtl_stride); + + /* Spectra -------------------------------------------------------------- */ + spectra_e_min = toml::find_or(toml_data, + "output", + "spectra", + "e_min", + defaults::output::spec_emin); + spectra_e_max = toml::find_or(toml_data, + "output", + "spectra", + "e_max", + defaults::output::spec_emax); + spectra_log_bins = toml::find_or(toml_data, + "output", + "spectra", + "log_bins", + defaults::output::spec_log); + spectra_n_bins = toml::find_or(toml_data, + "output", + "spectra", + "n_bins", + defaults::output::spec_nbins); + + /* Stats ---------------------------------------------------------------- */ + stats_quantities = toml::find_or(toml_data, + "output", + "stats", + "quantities", + defaults::output::stats_quantities); + stats_custom_quantities = toml::find_or(toml_data, + "output", + "stats", + "custom", + std::vector {}); + + /* Debug ---------------------------------------------------------------- */ + debug_as_is = toml::find_or(toml_data, "output", "debug", "as_is", false); + debug_ghosts = toml::find_or(toml_data, "output", "debug", "ghosts", false); + if (debug_ghosts) { + for (const auto& dwn : fields_downsampling) { + raise::ErrorIf( + dwn != 1, + "full resolution required when outputting with ghost cells", + HERE); + } + } + } + + void Output::setParams(SimulationParams* params) const { + params->set("output.format", format); + params->set("output.interval", global_interval); + params->set("output.interval_time", global_interval_time); + for (const auto& [category, cat_params] : categories) { + params->set("output." + category + ".enable", cat_params.enable); + params->set("output." + category + ".interval", cat_params.interval); + params->set("output." + category + ".interval_time", + cat_params.interval_time); + } + + params->set("output.fields.quantities", fields_quantities); + params->set("output.fields.custom", fields_custom_quantities); + params->set("output.fields.mom_smooth", fields_mom_smooth); + params->set("output.fields.downsampling", fields_downsampling); + + params->set("output.particles.species", particles_species); + params->set("output.particles.stride", particles_stride); + + params->set("output.spectra.e_min", spectra_e_min); + params->set("output.spectra.e_max", spectra_e_max); + params->set("output.spectra.log_bins", spectra_log_bins); + params->set("output.spectra.n_bins", spectra_n_bins); + + params->set("output.stats.quantities", stats_quantities); + params->set("output.stats.custom", stats_custom_quantities); + + params->set("output.debug.as_is", debug_as_is); + params->set("output.debug.ghosts", debug_ghosts); + } + + } // namespace params +} // namespace ntt diff --git a/src/framework/parameters/output.h b/src/framework/parameters/output.h new file mode 100644 index 000000000..e301fc745 --- /dev/null +++ b/src/framework/parameters/output.h @@ -0,0 +1,68 @@ +/** + * @file framework/parameters/output.h + * @brief Auxiliary functions for reading in output parameters + * @implements + * - ntt::params::Output + * - ntt::params::OutputCategory + * @cpp: + * - output.cpp + * @namespaces: + * - ntt::params:: + */ +#ifndef FRAMEWORK_PARAMETERS_OUTPUT_H +#define FRAMEWORK_PARAMETERS_OUTPUT_H + +#include "global.h" + +#include "framework/parameters/parameters.h" + +#include + +#include +#include +#include + +namespace ntt { + namespace params { + + struct OutputCategory { + bool enable; + timestep_t interval; + simtime_t interval_time; + }; + + struct Output { + std::string format; + + timestep_t global_interval; + simtime_t global_interval_time; + + std::map categories; + + std::vector fields_quantities; + std::vector fields_custom_quantities; + unsigned short fields_mom_smooth; + std::vector fields_downsampling; + + std::vector particles_species; + npart_t particles_stride; + + real_t spectra_e_min; + real_t spectra_e_max; + bool spectra_log_bins; + std::size_t spectra_n_bins; + + std::vector stats_quantities; + std::vector stats_custom_quantities; + + bool debug_as_is; + bool debug_ghosts; + + void read(Dimension, std::size_t, const toml::value&); + void setParams(SimulationParams*) const; + }; + + } // namespace params +} // namespace ntt + +#endif // FRAMEWORK_PARAMETERS_OUTPUT_H diff --git a/src/framework/parameters/parameters.cpp b/src/framework/parameters/parameters.cpp new file mode 100644 index 000000000..085e6bc39 --- /dev/null +++ b/src/framework/parameters/parameters.cpp @@ -0,0 +1,364 @@ +#include "framework/parameters/parameters.h" + +#include "defaults.h" +#include "enums.h" +#include "global.h" + +#include "utils/error.h" +#include "utils/formatting.h" +#include "utils/numeric.h" + +#include "framework/containers/species.h" +#include "framework/parameters/algorithms.h" +#include "framework/parameters/extra.h" +#include "framework/parameters/grid.h" +#include "framework/parameters/output.h" +#include "framework/parameters/particles.h" + +#include + +#if defined(MPI_ENABLED) + #include +#endif + +#include +#include +#include +#include +#include + +namespace ntt { + + /* + * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + * Parameters that must not be changed during the checkpoint restart + * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + */ + void SimulationParams::setImmutableParams(const toml::value& toml_data) { + /* [simulation] --------------------------------------------------------- */ + const auto engine_enum = SimEngine::pick( + fmt::toLower(toml::find(toml_data, "simulation", "engine")).c_str()); + set("simulation.engine", engine_enum); + + /* grid and decomposition ------------------------------------------------ */ + params::Grid grid_params {}; + grid_params.read(engine_enum, toml_data); + grid_params.setParams(this); + + /* [scales] ------------------------------------------------------------- */ + const auto larmor0 = toml::find(toml_data, "scales", "larmor0"); + const auto skindepth0 = toml::find(toml_data, "scales", "skindepth0"); + raise::ErrorIf(larmor0 <= ZERO || skindepth0 <= ZERO, + "larmor0 and skindepth0 must be positive", + HERE); + set("scales.larmor0", larmor0); + set("scales.skindepth0", skindepth0); + set("scales.sigma0", SQR(skindepth0 / larmor0)); + set("scales.B0", ONE / larmor0); + set("scales.omegaB0", ONE / larmor0); + + /* [particles] ---------------------------------------------------------- */ + const auto ppc0 = toml::find(toml_data, "particles", "ppc0"); + set("particles.ppc0", ppc0); + raise::ErrorIf(ppc0 <= 0.0, "ppc0 must be positive", HERE); + set("particles.use_weights", + toml::find_or(toml_data, "particles", "use_weights", false)); + const auto global_clearing_interval = toml::find_or( + toml_data, + "particles", + "clear_interval", + defaults::clear_interval); + set("particles.clear_interval", global_clearing_interval); + const auto global_spatial_sorting_interval = toml::find_or( + toml_data, + "particles", + "spatial_sorting_interval", + 0u); + set("particles.spatial_sorting_interval", global_spatial_sorting_interval); + + set("scales.n0", ppc0 / get("scales.V0")); + set("scales.q0", get("scales.V0") / (ppc0 * SQR(skindepth0))); + + /* [particles.species] -------------------------------------------------- */ + std::vector species; + const auto species_tab = toml::find_or(toml_data, + "particles", + "species", + toml::array {}); + set("particles.nspec", species_tab.size()); + + spidx_t idx = 1; + for (const auto& sp : species_tab) { + species.emplace_back( + params::GetParticleSpecies(this, + engine_enum, + idx, + sp, + global_clearing_interval, + global_spatial_sorting_interval)); + idx += 1; + } + set("particles.species", species); + } + + /* + * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + * Parameters that may be changed during the checkpoint restart + * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + */ + void SimulationParams::setMutableParams(const toml::value& toml_data) { + const auto engine_enum = get("simulation.engine"); + const auto coord_enum = get("grid.metric.coord"); + const auto dim = get("grid.dim"); + const auto extent_pairwise = get>("grid.extent"); + + /* [simulation] --------------------------------------------------------- */ + set("simulation.name", + toml::find(toml_data, "simulation", "name")); + set("simulation.runtime", + toml::find(toml_data, "simulation", "runtime")); + + /* [grid.boundaraies] --------------------------------------------------- */ + const auto [flds_bc, prtl_bc] = params::GetBoundaryConditions(this, + engine_enum, + dim, + coord_enum, + toml_data); + set("grid.boundaries.fields", flds_bc); + set("grid.boundaries.particles", prtl_bc); + + /* [particles] ---------------------------------------------------------- */ + const auto species_tab = toml::find_or(toml_data, + "particles", + "species", + toml::array {}); + std::vector species = get>( + "particles.species"); + raise::ErrorIf(species_tab.size() != species.size(), + "number of species changed after restart", + HERE); + + std::vector new_species; + + spidx_t idxM1 = 0; + for (const auto& sp : species_tab) { + const auto maxnpart_real = toml::find(sp, "maxnpart"); + const auto maxnpart = static_cast(maxnpart_real); + const auto particle_species = species[idxM1]; + new_species.emplace_back(particle_species.index(), + particle_species.label(), + particle_species.mass(), + particle_species.charge(), + maxnpart, + particle_species.clearing_interval(), + particle_species.spatial_sorting_interval(), + particle_species.pusher(), + particle_species.use_tracking(), + particle_species.radiative_drag_flags(), + particle_species.emission_policy_flag(), + particle_species.npld_r(), + particle_species.npld_i()); + idxM1++; + } + set("particles.species", new_species); + + /* [output] ------------------------------------------------------------- */ + params::Output output_params; + output_params.read(dim, get("particles.nspec"), toml_data); + output_params.setParams(this); + + /* [checkpoint] --------------------------------------------------------- */ + set("checkpoint.interval", + toml::find_or(toml_data, + "checkpoint", + "interval", + defaults::checkpoint::interval)); + set("checkpoint.interval_time", + toml::find_or(toml_data, "checkpoint", "interval_time", -1.0)); + set("checkpoint.keep", + toml::find_or(toml_data, "checkpoint", "keep", defaults::checkpoint::keep)); + auto walltime_str = toml::find_or(toml_data, + "checkpoint", + "walltime", + defaults::checkpoint::walltime); + if (walltime_str.empty()) { + walltime_str = defaults::checkpoint::walltime; + } + set("checkpoint.walltime", walltime_str); + + const auto checkpoint_write_path = toml::find_or( + toml_data, + "checkpoint", + "write_path", + fmt::format(defaults::checkpoint::write_path.c_str(), + get("simulation.name").c_str())); + set("checkpoint.write_path", checkpoint_write_path); + set("checkpoint.read_path", + toml::find_or(toml_data, "checkpoint", "read_path", checkpoint_write_path)); + + /* [diagnostics] -------------------------------------------------------- */ + set("diagnostics.interval", + toml::find_or(toml_data, "diagnostics", "interval", defaults::diag::interval)); + set("diagnostics.blocking_timers", + toml::find_or(toml_data, "diagnostics", "blocking_timers", false)); + set("diagnostics.colored_stdout", + toml::find_or(toml_data, "diagnostics", "colored_stdout", false)); + set("diagnostics.log_level", + toml::find_or(toml_data, "diagnostics", "log_level", defaults::diag::log_level)); + + /* inferred variables --------------------------------------------------- */ + params::Boundaries boundaries_params { + isPromised("grid.boundaries.match.ds"), + isPromised("grid.boundaries.absorb.ds"), + isPromised("grid.boundaries.atmosphere.temperature") + }; + boundaries_params.read(dim, coord_enum, extent_pairwise, toml_data); + boundaries_params.setParams(this); + + /* [algorithms] --------------------------------------------------------- */ + params::Algorithms alg_params {}; + std::map alg_extra_flags = { + { "gr", engine_enum == SimEngine::GRPIC }, + { "gca", isPromised("algorithms.gca.e_ovr_b_max") }, + }; + alg_params.read(get("scales.dx0"), alg_extra_flags, toml_data); + alg_params.setParams(alg_extra_flags, this); + + /* extra physics ------------------------------------------------------ */ + params::Extra extra_params {}; + std::map extra_extra_flags = { + { "synchrotron_drag",isPromised("radiation.drag.synchrotron.gamma_rad") }, + { "compton_drag", isPromised("radiation.drag.compton.gamma_rad") }, + { "synchrotron_emission", + isPromised("radiation.emission.synchrotron.photon_species") }, + { "compton_emission", isPromised("radiation.emission.compton.photon_species") } + }; + extra_params.read(extra_extra_flags, toml_data, this); + extra_params.setParams(extra_extra_flags, this); + + // @TODO: disabling stats for non-Cartesian + if (coord_enum != Coord::Cart) { + set("output.stats.enable", false); + } + } + + void SimulationParams::setSetupParams(const toml::value& toml_data) { + /* [setup] -------------------------------------------------------------- */ + const auto setup = toml::find_or(toml_data, "setup", toml::table {}); + for (const auto& [key, val] : setup) { + if (val.is_boolean()) { + set("setup." + key, (bool)(val.as_boolean())); + } else if (val.is_integer()) { + set("setup." + key, (int)(val.as_integer())); + } else if (val.is_floating()) { + set("setup." + key, (real_t)(val.as_floating())); + } else if (val.is_string()) { + set("setup." + key, (std::string)(val.as_string())); + } else if (val.is_array()) { + const auto val_arr = val.as_array(); + if (val_arr.size() == 0) { + continue; + } else { + if (val_arr[0].is_integer()) { + std::vector vec; + for (const auto& v : val_arr) { + vec.push_back(v.as_integer()); + } + set("setup." + key, vec); + } else if (val_arr[0].is_floating()) { + std::vector vec; + for (const auto& v : val_arr) { + vec.push_back(v.as_floating()); + } + set("setup." + key, vec); + } else if (val_arr[0].is_boolean()) { + std::vector vec; + for (const auto& v : val_arr) { + vec.push_back(v.as_boolean()); + } + set("setup." + key, vec); + } else if (val_arr[0].is_string()) { + std::vector vec; + for (const auto& v : val_arr) { + vec.push_back(v.as_string()); + } + set("setup." + key, vec); + } else if (val_arr[0].is_array()) { + if (val_arr[0].as_array().size() == 0) { + raise::Error("empty inner arrays not allowed in [setup]", HERE); + } else if (val_arr[0][0].is_integer()) { + std::vector> vec; + for (const auto& v1 : val_arr) { + std::vector inner_vec; + for (const auto& v2 : v1.as_array()) { + inner_vec.push_back(v2.as_integer()); + } + vec.push_back(inner_vec); + } + set("setup." + key, vec); + } else if (val_arr[0][0].is_floating()) { + std::vector> vec; + for (const auto& v1 : val_arr) { + std::vector inner_vec; + for (const auto& v2 : v1.as_array()) { + inner_vec.push_back(v2.as_floating()); + } + vec.push_back(inner_vec); + } + set("setup." + key, vec); + } else if (val_arr[0][0].is_boolean()) { + std::vector> vec; + for (const auto& v : val_arr) { + std::vector inner_vec; + for (const auto& v2 : v.as_array()) { + inner_vec.push_back(v2.as_boolean()); + } + vec.push_back(inner_vec); + } + set("setup." + key, vec); + } else if (val_arr[0][0].is_string()) { + std::vector> vec; + for (const auto& v : val_arr) { + std::vector inner_vec; + for (const auto& v2 : v.as_array()) { + inner_vec.push_back(v2.as_string()); + } + vec.push_back(inner_vec); + } + set("setup." + key, vec); + } else if (val_arr[0][0].is_array()) { + raise::Error("up to 2D arrays allowed in [setup]", HERE); + } + } else { + raise::Error("invalid setup variable type", HERE); + } + } + } + } + } + + void SimulationParams::setCheckpointParams(bool is_resuming, + timestep_t start_step, + simtime_t start_time) { + set("checkpoint.is_resuming", is_resuming); + set("checkpoint.start_step", start_step); + set("checkpoint.start_time", start_time); + } + + void SimulationParams::checkPromises() const { + raise::ErrorIf(!promisesFulfilled(), + "Have not defined all the necessary variables", + HERE); + } + + void SimulationParams::saveTOML(const std::string& path, simtime_t time) const { + CallOnce([&]() { + std::ofstream metadata; + metadata.open(path); + metadata << fmt::format("[metadata]\n time = %f\n\n", time) << data() + << std::endl; + metadata.close(); + }); + } + +} // namespace ntt diff --git a/src/framework/parameters.h b/src/framework/parameters/parameters.h similarity index 88% rename from src/framework/parameters.h rename to src/framework/parameters/parameters.h index 0af4fa405..a9eadc042 100644 --- a/src/framework/parameters.h +++ b/src/framework/parameters/parameters.h @@ -1,5 +1,5 @@ /** - * @file framework/parameters.h + * @file framework/parameters/parameters.h * @brief Structure for defining and holding initial simulation parameters * @implements * - ntt::SimulationParams : ntt::Parameters @@ -14,11 +14,12 @@ * @note A proper metric is used to infer the minimum cell size/volume etc. */ -#ifndef FRAMEWORK_PARAMETERS_H -#define FRAMEWORK_PARAMETERS_H +#ifndef FRAMEWORK_PARAMETERS_PARAMETERS_H +#define FRAMEWORK_PARAMETERS_PARAMETERS_H #include "utils/param_container.h" -#include "utils/toml.h" + +#include #include @@ -62,4 +63,4 @@ namespace ntt { } // namespace ntt -#endif // FRAMEWORK_PARAMETERS_H +#endif // FRAMEWORK_PARAMETERS_PARAMETERS_H diff --git a/src/framework/parameters/particles.cpp b/src/framework/parameters/particles.cpp new file mode 100644 index 000000000..afc795a9a --- /dev/null +++ b/src/framework/parameters/particles.cpp @@ -0,0 +1,228 @@ +#include "framework/parameters/particles.h" + +#include "defaults.h" +#include "enums.h" +#include "global.h" + +#include "utils/error.h" +#include "utils/formatting.h" + +#include "framework/containers/species.h" +#include "framework/parameters/parameters.h" + +#include + +#include + +namespace ntt { + + namespace params { + + /* + * Auxiliary functions + */ + auto getRadiativeDragFlags(const std::string& radiative_drag_str) + -> RadiativeDragFlags { + if (fmt::toLower(radiative_drag_str) == "none") { + return RadiativeDrag::NONE; + } else { + // separate comas + RadiativeDragFlags flags = RadiativeDrag::NONE; + std::string token; + std::istringstream tokenStream(radiative_drag_str); + while (std::getline(tokenStream, token, ',')) { + const auto token_lower = fmt::toLower(token); + if (token_lower == "synchrotron") { + flags |= RadiativeDrag::SYNCHROTRON; + } else if (token_lower == "compton") { + flags |= RadiativeDrag::COMPTON; + } else { + raise::Error(fmt::format("Invalid radiative_drag value: %s", + radiative_drag_str.c_str()), + HERE); + } + } + return flags; + } + } + + auto getPusherFlags(const std::string& particle_pusher_str) + -> ParticlePusherFlags { + if (fmt::toLower(particle_pusher_str) == "none") { + return ParticlePusher::NONE; + } else { + // separate comas + ParticlePusherFlags flags = ParticlePusher::NONE; + std::string token; + std::istringstream tokenStream(particle_pusher_str); + while (std::getline(tokenStream, token, ',')) { + const auto token_lower = fmt::toLower(token); + if (token_lower == "photon") { + flags |= ParticlePusher::PHOTON; + } else if (token_lower == "boris") { + flags |= ParticlePusher::BORIS; + } else if (token_lower == "vay") { + flags |= ParticlePusher::VAY; + } else if (token_lower == "gca") { + flags |= ParticlePusher::GCA; + } else { + raise::Error(fmt::format("Invalid pusher value: %s", + particle_pusher_str.c_str()), + HERE); + } + } + if (flags & ParticlePusher::PHOTON and flags & ParticlePusher::GCA) { + raise::Error("Photon pusher cannot be used with GCA", HERE); + } + return flags; + } + } + + auto getEmissionPolicyFlag(const std::string& emission_policy_str) + -> EmissionTypeFlag { + if (fmt::toLower(emission_policy_str) == "none") { + return EmissionType::NONE; + } else if (fmt::toLower(emission_policy_str) == "synchrotron") { + return EmissionType::SYNCHROTRON; + } else if (fmt::toLower(emission_policy_str) == "compton") { + return EmissionType::COMPTON; + } else if (fmt::toLower(emission_policy_str) == "custom") { + return EmissionType::CUSTOM; + } else { + raise::Error(fmt::format("Invalid emission_policy value: %s", + emission_policy_str.c_str()), + HERE); + return EmissionType::NONE; + } + } + + auto GetParticleSpecies(SimulationParams* params, + const SimEngine& engine_enum, + spidx_t idx, + const toml::value& sp, + timestep_t global_clearing_interval, + timestep_t global_spatial_sorting_interval) + -> ParticleSpecies { + const auto label = toml::find_or(sp, + "label", + "s" + std::to_string(idx)); + const auto mass = toml::find(sp, "mass"); + const auto charge = toml::find(sp, "charge"); + raise::ErrorIf((charge != 0.0f) && (mass == 0.0f), + "mass of the charged species must be non-zero", + HERE); + const auto is_massless = (mass == 0.0f) && (charge == 0.0f); + const auto def_pusher = (is_massless ? defaults::ph_pusher + : defaults::em_pusher); + const auto maxnpart_real = toml::find(sp, "maxnpart"); + const auto maxnpart = static_cast(maxnpart_real); + const auto clearing_interval = toml::find_or( + sp, + "clear_interval", + global_clearing_interval); + const auto spatial_sorting_interval = toml::find_or( + sp, + "spatial_sorting_interval", + global_spatial_sorting_interval); + auto pusher_str = toml::find_or(sp, "pusher", std::string(def_pusher)); + const auto npayloads_real = toml::find_or(sp, + "n_payloads_real", + static_cast(0)); + const auto use_tracking = toml::find_or(sp, "tracking", false); + auto npayloads_int = toml::find_or(sp, + "n_payloads_int", + static_cast(0)); + if (use_tracking) { +#if !defined(MPI_ENABLED) + npayloads_int += 1; +#else + npayloads_int += 2; +#endif + } + auto radiative_drag_str = toml::find_or(sp, + "radiative_drag", + std::string("default")); + + const auto radiative_drag_defaulted = (fmt::toLower(radiative_drag_str) == + "default"); + if (radiative_drag_defaulted) { + radiative_drag_str = "none"; + } + + const auto emission_policy_str = toml::find_or(sp, + "emission", + std::string("none")); + raise::ErrorIf((fmt::toLower(radiative_drag_str) != "none") && is_massless, + "radiative drag is only applicable to massive particles", + HERE); + raise::ErrorIf((fmt::toLower(pusher_str) == "photon") && !is_massless, + "photon pusher is only applicable to massless particles", + HERE); + + auto particle_pusher_flags = getPusherFlags(pusher_str); + auto radiative_drag_flags = getRadiativeDragFlags(radiative_drag_str); + auto emission_policy_flag = getEmissionPolicyFlag(emission_policy_str); + + raise::ErrorIf((emission_policy_flag == EmissionType::SYNCHROTRON or + emission_policy_flag == EmissionType::COMPTON) and + is_massless, + "Radiative emission policies are only applicable to " + "massive particles", + HERE); + + if (radiative_drag_defaulted) { + if (emission_policy_flag == EmissionType::SYNCHROTRON) { + radiative_drag_flags |= RadiativeDrag::SYNCHROTRON; + } else if (emission_policy_flag == EmissionType::COMPTON) { + radiative_drag_flags |= RadiativeDrag::COMPTON; + } + } + + if (radiative_drag_flags & RadiativeDrag::SYNCHROTRON) { + raise::ErrorIf(engine_enum != SimEngine::SRPIC, + "Synchrotron radiative drag is only supported for SRPIC", + HERE); + params->promiseToDefine("radiation.drag.synchrotron.gamma_rad"); + } + if (radiative_drag_flags & RadiativeDrag::COMPTON) { + raise::ErrorIf( + engine_enum != SimEngine::SRPIC, + "Inverse Compton radiative drag is only supported for SRPIC", + HERE); + params->promiseToDefine("radiation.drag.compton.gamma_rad"); + } + if (particle_pusher_flags & ParticlePusher::GCA) { + raise::ErrorIf(engine_enum != SimEngine::SRPIC, + "GCA pushers are only supported for SRPIC", + HERE); + params->promiseToDefine("algorithms.gca.e_ovr_b_max"); + params->promiseToDefine("algorithms.gca.larmor_max"); + } + + if (emission_policy_flag == EmissionType::SYNCHROTRON) { + params->promiseToDefine( + "radiation.emission.synchrotron.photon_species"); + params->promiseToDefine("radiation.drag.synchrotron.gamma_rad"); + } else if (emission_policy_flag == EmissionType::COMPTON) { + params->promiseToDefine("radiation.emission.compton.photon_species"); + params->promiseToDefine("radiation.drag.compton.gamma_rad"); + } + + return ParticleSpecies(idx, + label, + mass, + charge, + maxnpart, + clearing_interval, + spatial_sorting_interval, + particle_pusher_flags, + use_tracking, + radiative_drag_flags, + emission_policy_flag, + npayloads_real, + npayloads_int); + } + + } // namespace params + +} // namespace ntt diff --git a/src/framework/parameters/particles.h b/src/framework/parameters/particles.h new file mode 100644 index 000000000..21ce303ba --- /dev/null +++ b/src/framework/parameters/particles.h @@ -0,0 +1,36 @@ +/** + * @file framework/parameters/particles.h + * @brief Auxiliary functions for reading in particle species parameters + * @implements + * - ntt::params::GetParticleSpecies -> ParticleSpecies + * @cpp: + * - particles.cpp + * @namespaces: + * - ntt::params:: + */ +#ifndef FRAMEWORK_PARAMETERS_PARTICLES_H +#define FRAMEWORK_PARAMETERS_PARTICLES_H + +#include "enums.h" + +#include "framework/containers/species.h" +#include "framework/parameters/parameters.h" + +#include + +namespace ntt { + + namespace params { + + auto GetParticleSpecies(SimulationParams*, + const SimEngine&, + spidx_t, + const toml::value&, + timestep_t, + timestep_t) -> ParticleSpecies; + + } // namespace params + +} // namespace ntt + +#endif diff --git a/src/framework/simulation.cpp b/src/framework/simulation.cpp index 20dc39486..4e0da3eda 100644 --- a/src/framework/simulation.cpp +++ b/src/framework/simulation.cpp @@ -9,7 +9,8 @@ #include "utils/formatting.h" #include "utils/log.h" #include "utils/plog.h" -#include "utils/toml.h" + +#include #include #include diff --git a/src/framework/simulation.h b/src/framework/simulation.h index 33750030f..7387d62dd 100644 --- a/src/framework/simulation.h +++ b/src/framework/simulation.h @@ -16,11 +16,12 @@ #include "enums.h" -#include "arch/traits.h" #include "utils/error.h" -#include "utils/toml.h" -#include "framework/parameters.h" +#include "engines/traits.h" +#include "framework/parameters/parameters.h" + +#include namespace ntt { @@ -36,12 +37,9 @@ namespace ntt { ~Simulation(); template