binding wrapper: CI/build fix, add binding stubs

CI:
- Python unit and integration tests have their own CI test now.
- Unit and integration test pipelines reverted to pre-binding related
  changes. Binding need workarounds to comply with packaging, CI
  and building compatibility.
- Format fixes.
- `binding.py` (binding wrapper) and bindings have stubs now.

CMake build:
- Naming is more straight forward.
- Bindings install into `pythonX.Y/site-packages/villas/node/`.
  Should be compatible with a potential pypi-package related to the
  wrapper binding.

Signed-off-by: Kevin Vu te Laar <vu.te@rwth-aachen.de>

Author: Kevin Vu te Laar

.gitlab-ci.yml

@@ -168,13 +168,34 @@ test:cppcheck:
       - cppcheck.log
     expose_as: cppcheck
 
+test:python_unit_integration:
+  stage: test
+  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
+  before_script:
+    # dependency from node.py, which gets imported because of __init__.py in node/villas/python/node
+    - pip install requests
+  script:
+    # build binding and symlink to villas.node folder
+    # helps with correct imports and compatiblity for CI, binding wrapper and binding install
+    - export PYTHONPATH=$PYTHONPATH:${PWD}/python
+    - cmake --build build ${CMAKE_BUILD_OPTS} --target python_binding
+
+    - binding_path=$(find ${PWD}/build/python/binding/ -name "python_binding*.so" | head -n1)
+    - link_path=${PWD}/python/villas/node/$(basename $python_binding)
+    - ln -sf $binding_path $link_path
+
+    - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-unit-tests run-python-integration-tests
+    - rm $node_path
+  needs:
+    - job: "build:source: [fedora]"
+      artifacts: true
+
 test:unit:
   stage: test
   image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
   script:
     - cmake -S . -B build ${CMAKE_OPTS}
-    - export PYTHONPATH=$PYTHONPATH:${PWD}/build/python/binding/:${PWD}/python/villas/node/
-    - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common run-python-unit-tests
+    - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common
   needs:
     - job: "build:source: [fedora]"
       artifacts: true
@@ -184,8 +205,7 @@ test:integration:
   image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
   script:
     - cmake -S . -B build ${CMAKE_OPTS}
-    - export PYTHONPATH=$PYTHONPATH:${PWD}/build/python/binding/:${PWD}/python/villas/node/
-    - cmake --build build ${CMAKE_BUILD_OPTS} --target run-integration-tests run-python-integration-tests
+    - cmake --build build ${CMAKE_BUILD_OPTS} --target run-integration-tests
   artifacts:
     name: ${CI_PROJECT_NAME}-integration-tests-${CI_BUILD_REF}
     when: always

python/binding/CMakeLists.txt

@@ -11,25 +11,29 @@ if(pybind11_FOUND)
   find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
 
   execute_process(
-    COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('stdlib') + '/lib-dynload')"
-    OUTPUT_VARIABLE PYTHON_LIB_DYNLOAD_DIR
+    COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('purelib'))"
+    OUTPUT_VARIABLE PYTHON_SITE_PACKAGES
     OUTPUT_STRIP_TRAILING_WHITESPACE
   )
 
   message(STATUS "Found Python version: ${Python_VERSION}")
   message(STATUS "Python major version: ${Python_VERSION_MAJOR}")
   message(STATUS "Python minor version: ${Python_VERSION_MINOR}")
-  message(STATUS "Python .so install directory: ${PYTHON_LIB_DYNLOAD_DIR}")
 
-  pybind11_add_module(python-binding villas-python-binding.cpp)
-  set_target_properties(python-binding PROPERTIES OUTPUT_NAME villas_node)
-  target_link_libraries(python-binding PUBLIC villas)
+  pybind11_add_module(python_binding capi_python_binding.cpp)
+  target_link_libraries(python_binding PUBLIC villas)
 
   install(
-    TARGETS python-binding
+    TARGETS python_binding
     COMPONENT lib
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${PYTHON_LIB_DYNLOAD_DIR}
+    LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}/villas/node/
+  )
+  file(WRITE "${CMAKE_BINARY_DIR}/python/villas/__init__.py" "")
+  file(WRITE "${CMAKE_BINARY_DIR}/python/villas/node/__init__.py" "")
+  install(
+    DIRECTORY "${CMAKE_BINARY_DIR}/python/villas"
+    DESTINATION "${PYTHON_SITE_PACKAGES}"
+    FILES_MATCHING PATTERN "__init__.py"
   )
 else()
   message(STATUS "pybind11 not found. Skipping Python wrapper build.")

python/binding/villas-python-binding.cpp python/binding/capi_python_binding.cpppython/binding/villas-python-binding.cpp renamed to python/binding/capi_python_binding.cpp

@@ -67,7 +67,7 @@ class Array {
  * @param villas_node   Name of the module to be bound
  * @param m             Access variable for modifying the module code
  */
-PYBIND11_MODULE(villas_node, m) {
+PYBIND11_MODULE(python_binding, m) {
   m.def("memory_init", &memory_init);
 
   m.def("node_check", [](void *n) -> int { return node_check((vnode *)n); });

python/villas/node/binding.py

@@ -10,7 +10,7 @@
 import json
 import logging
 
-import villas_node as vn
+import villas.node.python_binding as vn
 
 logger = logging.getLogger("villas.node")
 

python/villas/node/binding.pyi

@@ -0,0 +1,48 @@
+from _typeshed import Incomplete
+from typing import Any, Callable
+
+logger: Incomplete
+
+
+class SamplesArray:
+    def __init__(self, length: int) -> None: ...
+    def __len__(self) -> int: ...
+    def __getitem__(self, idx: int | slice): ...
+    def __copy__(self) -> None: ...
+    def __deepcopy__(self) -> None: ...
+
+
+def _warn_if_not_implemented(
+    func: Callable[..., Any],
+) -> Callable[..., Any]: ...
+def memory_init(*args): ...
+def node_check(node): ...
+def node_destroy(node): ...
+def node_details(node): ...
+def node_input_signals_max_cnt(node): ...
+def node_is_enabled(node): ...
+def node_is_valid_name(name: str): ...
+def node_name(node): ...
+def node_name_full(node): ...
+def node_name_short(node): ...
+def node_netem_fds(*args): ...
+def node_new(config, uuid: str | None = None): ...
+def node_output_signals_max_cnt(node): ...
+def node_pause(node): ...
+def node_poll_fds(*args): ...
+def node_prepare(node): ...
+@_warn_if_not_implemented
+def node_read(node, samples, sample_length, count): ...
+def node_restart(node): ...
+def node_resume(node): ...
+@_warn_if_not_implemented
+def node_reverse(node): ...
+def node_start(node): ...
+def node_stop(node): ...
+def node_to_json(node): ...
+def node_to_json_str(node): ...
+@_warn_if_not_implemented
+def node_write(node, samples, count): ...
+def sample_length(*args) -> None: ...
+def sample_pack(*args) -> None: ...
+def sample_unpack(*args) -> None: ...

python/villas/node/python_binding.pyi

@@ -0,0 +1,63 @@
+import typing
+from typing import Any, Callable, overload
+
+Array = Any
+capsule = Any
+timespec = Any
+
+
+class SamplesArray:
+    def __init__(self, len: int) -> None: ...
+    def bulk_alloc(self, arg0: int, arg1: int, arg2: int) -> None: ...
+    def get_block(self, arg0: int) -> capsule: ...
+    def __getitem__(self, arg0: int) -> capsule: ...
+    def __iter__(self) -> typing.Iterator[capsule]: ...
+    def __setitem__(self, arg0: int, arg1: capsule) -> None: ...
+
+
+def memory_init(arg0: int) -> int: ...
+def node_check(arg0: capsule) -> int: ...
+def node_destroy(arg0: capsule) -> int: ...
+def node_details(arg0: capsule) -> str: ...
+def node_input_signals_max_cnt(arg0: capsule) -> int: ...
+def node_is_enabled(arg0: capsule) -> bool: ...
+def node_is_valid_name(arg0: str) -> bool: ...
+def node_name(arg0: capsule) -> str: ...
+def node_name_full(arg0: capsule) -> str: ...
+def node_name_short(arg0: capsule) -> str: ...
+def node_netem_fds(arg0: capsule, arg1: int) -> int: ...
+def node_new(arg0: str, arg1: str) -> capsule: ...
+def node_output_signals_max_cnt(arg0: capsule) -> int: ...
+def node_pause(arg0: capsule) -> int: ...
+def node_poll_fds(arg0: capsule, arg1: int) -> int: ...
+def node_prepare(arg0: capsule) -> int: ...
+@overload
+def node_read(arg0: capsule, arg1: Array, arg2: int) -> int: ...
+@overload
+def node_read(arg0: capsule, arg1: capsule, arg2: int) -> int: ...  # type: ignore[overload-cannot-match]
+def node_restart(arg0: capsule) -> int: ...
+def node_resume(arg0: capsule) -> int: ...
+def node_reverse(arg0: capsule) -> int: ...
+def node_start(arg0: capsule) -> int: ...
+def node_stop(arg0: capsule) -> int: ...
+def node_to_json_str(arg0: capsule) -> str: ...
+@overload
+def node_write(arg0: capsule, arg1: Array, arg2: int) -> int: ...
+@overload
+def node_write(arg0: capsule, arg1: capsule, arg2: int) -> int: ...  # type: ignore[overload-cannot-match]
+def sample_alloc(arg0: int) -> capsule: ...
+def sample_decref(arg0: capsule) -> None: ...
+def sample_length(arg0: capsule) -> int: ...
+def sample_pack(
+    arg0: int, arg1: timespec, arg2: timespec, arg3: int, arg4: float
+) -> capsule: ...
+def sample_unpack(
+    arg0: capsule,
+    arg1: int,
+    arg2: timespec,
+    arg3: timespec,
+    arg4: int,
+    arg5: int,
+    arg6: float,
+) -> None: ...
+def smps_array(arg0: int) -> Array: ...

tests/integration/CMakeLists.txt

@@ -22,7 +22,7 @@ add_custom_target(run-python-integration-tests
     COMMAND
         python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR}
     DEPENDS
-        python-binding
+        python_binding
     USES_TERMINAL
 )